From cc987a8e15edb50902213e8a498a1230115ac114 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 11:11:19 -0700 Subject: [PATCH 01/42] mag order and hit updates --- .../config/imap_ialirt_l1_variable_attrs.yaml | 87 ++++++++++++++++--- imap_processing/ialirt/utils/constants.py | 40 ++++----- imap_processing/ialirt/utils/create_xarray.py | 3 +- 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 5340694276..181bb370b0 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -189,17 +189,21 @@ codice_lo_fe_low_over_fe_high_ratio: hit_e_a_side_low_en: <<: *default_uint32 - CATDESC: Low energy (~300 keV) electrons (A-side) - FIELDNAM: Low energy (~300 keV) electrons (A-side) + CATDESC: Low energy (>0.5 MeV) electrons (A-side) + FIELDNAM: Low energy (>0.5 MeV) electrons (A-side) LABLAXIS: hit_e_a_side_low_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_e_a_side_med_en: <<: *default_uint32 - CATDESC: Medium energy (~3 MeV) electrons (A-side) - FIELDNAM: Medium energy (~3 MeV) electrons (A-side) + CATDESC: Medium energy (<1 MeV) electrons (A-side) + FIELDNAM: Medium energy (<1 MeV) electrons (A-side) LABLAXIS: hit_e_a_side_med_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_e_a_side_high_en: <<: *default_uint32 @@ -207,20 +211,26 @@ hit_e_a_side_high_en: FIELDNAM: High energy (>3 MeV) electrons (A-side) LABLAXIS: hit_e_a_side_high_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_e_b_side_low_en: <<: *default_uint32 - CATDESC: Low energy (~300 keV) electrons (B-side) - FIELDNAM: Low energy (~300 keV) electrons (B-side) + CATDESC: Low energy (>0.5 keV) electrons (B-side) + FIELDNAM: Low energy (>0.5 keV) electrons (B-side) LABLAXIS: hit_e_b_side_low_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_e_b_side_med_en: <<: *default_uint32 - CATDESC: Medium energy (~3 MeV) electrons (B-side) - FIELDNAM: Medium energy (~3 MeV) electrons (B-side) + CATDESC: Medium energy (<1 MeV) electrons (B-side) + FIELDNAM: Medium energy (<1 MeV) electrons (B-side) LABLAXIS: hit_e_b_side_med_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_e_b_side_high_en: <<: *default_uint32 @@ -228,13 +238,26 @@ hit_e_b_side_high_en: FIELDNAM: High energy (>3 MeV) electrons (B-side) LABLAXIS: hit_e_b_side_high_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 + +hit_h_omni_low_en: + <<: *default_uint32 + CATDESC: Low energy (6–8 MeV) protons (Omnidirectional) + FIELDNAM: Low energy (6–8 MeV) protons (Omnidirectional) + LABLAXIS: hit_h_omni_low_en + UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_h_omni_med_en: <<: *default_uint32 - CATDESC: Medium energy (12–70 MeV) protons (Omnidirectional) - FIELDNAM: Medium energy (12–70 MeV) protons (Omnidirectional) + CATDESC: Medium energy (12–15 MeV) protons (Omnidirectional) + FIELDNAM: Medium energy (12–15 MeV) protons (Omnidirectional) LABLAXIS: hit_h_omni_med_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_h_a_side_high_en: <<: *default_uint32 @@ -242,6 +265,8 @@ hit_h_a_side_high_en: FIELDNAM: High energy (>70 MeV) protons (A-side) LABLAXIS: hit_h_a_side_high_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_h_b_side_high_en: <<: *default_uint32 @@ -249,6 +274,8 @@ hit_h_b_side_high_en: FIELDNAM: High energy (>70 MeV) protons (B-side) LABLAXIS: hit_h_b_side_high_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_he_omni_low_en: <<: *default_uint32 @@ -256,6 +283,8 @@ hit_he_omni_low_en: FIELDNAM: Low energy (6–8 MeV/nuc) He (Omnidirectional) LABLAXIS: hit_he_omni_low_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 hit_he_omni_high_en: <<: *default_uint32 @@ -263,6 +292,8 @@ hit_he_omni_high_en: FIELDNAM: High energy (15–70 MeV/nuc) He (Omnidirectional) LABLAXIS: hit_he_omni_high_en UNITS: counts per second + VALIDMIN: 0 + VALIDMAX: 1000000000 mag_epoch: <<: *default_float32 @@ -348,6 +379,42 @@ mag_theta_B_GSE: VALIDMIN: -90 VALIDMAX: 90 +sc_position_GSM: + <<: *default_float32 + CATDESC: Spacecraft position in GSM coordinates + FIELDNAM: sc_position_GSM + UNITS: km + DEPEND_1: component + VALIDMIN: -3000000.0 + VALIDMAX: 3000000.0 + +sc_velocity_GSM: + <<: *default_float32 + CATDESC: Spacecraft velocity in GSM coordinates + FIELDNAM: sc_velocity_GSM + UNITS: km/s + DEPEND_1: component + VALIDMIN: -1000.0 + VALIDMAX: 1000.0 + +sc_position_GSE: + <<: *default_float32 + CATDESC: Spacecraft position in GSE coordinates + FIELDNAM: sc_position_GSE + UNITS: km + DEPEND_1: component + VALIDMIN: -3000000.0 + VALIDMAX: 3000000.0 + +sc_velocity_GSE: + <<: *default_float32 + CATDESC: Spacecraft velocity in GSE coordinates + FIELDNAM: sc_velocity_GSE + UNITS: km/s + DEPEND_1: component + VALIDMIN: -1000.0 + VALIDMAX: 1000.0 + swapi_pseudo_proton_density: <<: *default_float32 CATDESC: Pseudo density of solar wind protons diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index a8264be80c..934ffe1e13 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -15,46 +15,40 @@ "codice_lo_o_plus_7_over_o_plus_6_ratio", # Fe low/Fe high charge state ratio "codice_lo_fe_low_over_fe_high_ratio", - # Low energy (~300 keV) electrons (A-side) + # Low energy (>0.5 MeV) electrons (A-side) "hit_e_a_side_low_en", - # Medium energy (~3 MeV) electrons (A-side) + # Medium energy (<1 MeV) electrons (A-side) "hit_e_a_side_med_en", - # High energy (>3 MeV) electrons (A-side) - "hit_e_a_side_high_en", - # Low energy (~300 keV) electrons (B-side) + # Low energy (>0.5 MeV) electrons (B-side) "hit_e_b_side_low_en", - # Medium energy (~3 MeV) electrons (B-side) + # Medium energy (<1 MeV) electrons (B-side) "hit_e_b_side_med_en", - # High energy (>3 MeV) electrons (B-side) - "hit_e_b_side_high_en", - # Medium energy (12 to 70 MeV) protons (Omnidirectional) + # Low energy (6 to 8 MeV) protons (Omnidirectional) + "hit_h_omni_low_en", + # Medium energy (12 to 15 MeV) protons (Omnidirectional) "hit_h_omni_med_en", - # High energy (>70 MeV) protons (A-side) - "hit_h_a_side_high_en", - # High energy (>70 MeV) protons (B-side) - "hit_h_b_side_high_en", # Low energy (6 to 8 MeV/nuc) He (Omnidirectional) "hit_he_omni_low_en", # High energy (15 to 70 MeV/nuc) He (Omnidirectional) "hit_he_omni_high_en", # MAG instrument epoch "mag_epoch", + # Magnitude of the magnetic field vector + "mag_B_magnitude", + # Magnetic field vector in RTN coordinates + "mag_B_RTN", # Magnetic field vector in GSE coordinates "mag_B_GSE", + # Elevation angle (θ) of the magnetic field in GSE coordinates + "mag_theta_B_GSE", + # Azimuth angle (φ) of the magnetic field in GSE coordinates + "mag_phi_B_GSE", # Magnetic field vector in GSM coordinates "mag_B_GSM", - # Magnetic field vector in RTN coordinates - "mag_B_RTN", - # Magnitude of the magnetic field vector - "mag_B_magnitude", - # Azimuth angle (φ) of the magnetic field in GSM coordinates - "mag_phi_B_GSM", # Elevation angle (θ) of the magnetic field in GSM coordinates "mag_theta_B_GSM", - # Azimuth angle (φ) of the magnetic field in GSE coordinates - "mag_phi_B_GSE", - # Elevation angle (θ) of the magnetic field in GSE coordinates - "mag_theta_B_GSE", + # Azimuth angle (φ) of the magnetic field in GSM coordinates + "mag_phi_B_GSM", # Pseudo density of solar wind protons "swapi_pseudo_proton_density", # Pseudo speed of solar wind protons in solar inertial frame diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index fd28e9c6b3..d4e7db7afd 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -25,7 +25,6 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 cdf_manager.add_instrument_global_attrs("ialirt") cdf_manager.add_instrument_variable_attrs("ialirt", "l1") - instrument_keys: set[str] = set(IALIRT_KEYS) n = len(records) attrs = cdf_manager.get_variable_attributes("default_int64_attrs") fillval = attrs.get("FILLVAL") @@ -104,7 +103,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) # Create empty dataset for each key. - for key in instrument_keys: + for key in IALIRT_KEYS: attrs = cdf_manager.get_variable_attributes(key, check_schema=False) fillval = attrs.get("FILLVAL") if key in ["mag_B_GSE", "mag_B_GSM"]: From 96baa29d0f73a8e7cb3ac6ff34a0201a0fb39b08 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 11:26:37 -0700 Subject: [PATCH 02/42] update units --- imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 181bb370b0..39245b4b8f 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -216,8 +216,8 @@ hit_e_a_side_high_en: hit_e_b_side_low_en: <<: *default_uint32 - CATDESC: Low energy (>0.5 keV) electrons (B-side) - FIELDNAM: Low energy (>0.5 keV) electrons (B-side) + CATDESC: Low energy (>0.5 MeV) electrons (B-side) + FIELDNAM: Low energy (>0.5 MeV) electrons (B-side) LABLAXIS: hit_e_b_side_low_en UNITS: counts per second VALIDMIN: 0 From 0cfb7db32ec69362c988fc81cb912edc565f302c Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 12:25:32 -0700 Subject: [PATCH 03/42] update codice and lablaxis --- .../config/imap_ialirt_l1_variable_attrs.yaml | 224 ++++++++++-------- imap_processing/ialirt/utils/create_xarray.py | 43 ++-- 2 files changed, 146 insertions(+), 121 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 39245b4b8f..6dc7dfe06f 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -11,57 +11,67 @@ default_attrs: &default component: CATDESC: Component direction (x, y, z) - FIELDNAM: component - LABLAXIS: component + FIELDNAM: Vector component + LABLAXIS: Component FORMAT: A3 VAR_TYPE: metadata UNITS: " " RTN_component: CATDESC: Component direction (radial, tangential, normal) - FIELDNAM: RTN_component - LABLAXIS: RTN_component + FIELDNAM: RTN vector component + LABLAXIS: RTN component FORMAT: A3 VAR_TYPE: metadata UNITS: " " esa_step: CATDESC: ESA step number. The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. - FIELDNAM: esa_step - LABLAXIS: esa_step + FIELDNAM: ESA step index + LABLAXIS: ESA step FORMAT: I2 VALIDMIN: 0 VALIDMAX: 7 VAR_TYPE: metadata UNITS: " " -codice_hi_h_energy_ranges: - CATDESC: CoDICE-Hi energy ranges - FIELDNAM: codice_hi_h_energy_ranges - LABLAXIS: codice_hi_h_energy_ranges +codice_hi_h_spin_angle: + CATDESC: CoDICE-Hi spin angle bin + FIELDNAM: CoDICE-Hi spin angle bin + LABLAXIS: Spin angle bin + FORMAT: I3 + VALIDMIN: 0 + VALIDMAX: 3 + VAR_TYPE: metadata + UNITS: " " + +codice_hi_h_energy_range: + CATDESC: CoDICE-Hi H energy range + FIELDNAM: CoDICE-Hi H energy range + LABLAXIS: Energy range FORMAT: I3 VALIDMIN: 0 VALIDMAX: 14 VAR_TYPE: metadata UNITS: " " -codice_hi_h_elevation: - CATDESC: CoDICE-Hi elevation - FIELDNAM: codice_hi_h_elevation - LABLAXIS: codice_hi_h_elevation +codice_hi_h_spin_sector: + CATDESC: CoDICE-Hi spin sector + FIELDNAM: CoDICE-Hi spin sector + LABLAXIS: Spin sector FORMAT: I3 VALIDMIN: 0 - VALIDMAX: 4 + VALIDMAX: 3 VAR_TYPE: metadata UNITS: " " -codice_hi_h_spin_angle: - CATDESC: CoDICE-Hi spin_angle - FIELDNAM: codice_hi_h_spin_angle - LABLAXIS: codice_hi_h_spin_angle +codice_hi_h_azimuth: + CATDESC: CoDICE-Hi azimuth bin + FIELDNAM: CoDICE-Hi azimuth bin + LABLAXIS: Azimuth bin FORMAT: I3 VALIDMIN: 0 - VALIDMAX: 4 + VALIDMAX: 3 VAR_TYPE: metadata UNITS: " " @@ -123,21 +133,22 @@ default_float64_attrs: &default_float64 codice_hi_h: <<: *default_float32 - CATDESC: H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins - FIELDNAM: H intensities - LABLAXIS: Diff. Intensity + CATDESC: H intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins + FIELDNAM: H intensity + LABLAXIS: H intensity UNITS: "# / cm2-sr-s- MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 - DEPEND_1: codice_hi_h_energy_ranges - DEPEND_2: codice_hi_h_elevation - DEPEND_3: codice_hi_h_spin_angle + DEPEND_1: codice_hi_h_spin_angle + DEPEND_2: codice_hi_h_energy_range + DEPEND_3: codice_hi_h_spin_sector + DEPEND_4: codice_hi_h_azimuth codice_lo_c_over_o_abundance: <<: *default_float32 CATDESC: C/O abundance ratio - FIELDNAM: C/O - LABLAXIS: ratio + FIELDNAM: C/O abundance ratio + LABLAXIS: C/O ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -145,8 +156,8 @@ codice_lo_c_over_o_abundance: codice_lo_mg_over_o_abundance: <<: *default_float32 CATDESC: Mg/O abundance ratio - FIELDNAM: Mg/O - LABLAXIS: ratio + FIELDNAM: Mg/O abundance ratio + LABLAXIS: Mg/O ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -154,8 +165,8 @@ codice_lo_mg_over_o_abundance: codice_lo_fe_over_o_abundance: <<: *default_float32 CATDESC: Fe/O abundance ratio - FIELDNAM: Fe/O - LABLAXIS: ratio + FIELDNAM: Fe/O abundance ratio + LABLAXIS: Fe/O ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -163,8 +174,8 @@ codice_lo_fe_over_o_abundance: codice_lo_c_plus_6_over_c_plus_5_ratio: <<: *default_float32 CATDESC: C+6/C+5 charge state ratio - FIELDNAM: C+6/C+5 - LABLAXIS: ratio + FIELDNAM: C6+/C5+ charge ratio + LABLAXIS: C6+/C5+ UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -172,17 +183,17 @@ codice_lo_c_plus_6_over_c_plus_5_ratio: codice_lo_o_plus_7_over_o_plus_6_ratio: <<: *default_float32 CATDESC: O+7/O+6 charge state ratio - FIELDNAM: O+7/O+6 - LABLAXIS: ratio + FIELDNAM: O7+/O6+ charge ratio + LABLAXIS: O7+/O6+ UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 codice_lo_fe_low_over_fe_high_ratio: <<: *default_float32 - CATDESC: Fe low/Fe high charge state ratio, low charge state = 6-12, high charge state = 13-17 - FIELDNAM: Fe low/Fe high - LABLAXIS: ratio + CATDESC: Fe low/high charge state ratio (low=6–12, high=13–17) + FIELDNAM: Fe low/high charge ratio + LABLAXIS: Fe low/high UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -190,8 +201,8 @@ codice_lo_fe_low_over_fe_high_ratio: hit_e_a_side_low_en: <<: *default_uint32 CATDESC: Low energy (>0.5 MeV) electrons (A-side) - FIELDNAM: Low energy (>0.5 MeV) electrons (A-side) - LABLAXIS: hit_e_a_side_low_en + FIELDNAM: Electrons low energy (A-side) + LABLAXIS: e- low energy (A) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -199,8 +210,8 @@ hit_e_a_side_low_en: hit_e_a_side_med_en: <<: *default_uint32 CATDESC: Medium energy (<1 MeV) electrons (A-side) - FIELDNAM: Medium energy (<1 MeV) electrons (A-side) - LABLAXIS: hit_e_a_side_med_en + FIELDNAM: Electrons medium energy (A-side) + LABLAXIS: e- medium energy (A) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -208,8 +219,8 @@ hit_e_a_side_med_en: hit_e_a_side_high_en: <<: *default_uint32 CATDESC: High energy (>3 MeV) electrons (A-side) - FIELDNAM: High energy (>3 MeV) electrons (A-side) - LABLAXIS: hit_e_a_side_high_en + FIELDNAM: Electrons high energy (A-side) + LABLAXIS: e- high energy (A) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -217,8 +228,8 @@ hit_e_a_side_high_en: hit_e_b_side_low_en: <<: *default_uint32 CATDESC: Low energy (>0.5 MeV) electrons (B-side) - FIELDNAM: Low energy (>0.5 MeV) electrons (B-side) - LABLAXIS: hit_e_b_side_low_en + FIELDNAM: Electrons low energy (B-side) + LABLAXIS: e- low energy (B) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -226,8 +237,8 @@ hit_e_b_side_low_en: hit_e_b_side_med_en: <<: *default_uint32 CATDESC: Medium energy (<1 MeV) electrons (B-side) - FIELDNAM: Medium energy (<1 MeV) electrons (B-side) - LABLAXIS: hit_e_b_side_med_en + FIELDNAM: Electrons medium energy (B-side) + LABLAXIS: e- medium energy (B) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -235,26 +246,26 @@ hit_e_b_side_med_en: hit_e_b_side_high_en: <<: *default_uint32 CATDESC: High energy (>3 MeV) electrons (B-side) - FIELDNAM: High energy (>3 MeV) electrons (B-side) - LABLAXIS: hit_e_b_side_high_en + FIELDNAM: Electrons high energy (B-side) + LABLAXIS: e- high energy (B) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 hit_h_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6–8 MeV) protons (Omnidirectional) - FIELDNAM: Low energy (6–8 MeV) protons (Omnidirectional) - LABLAXIS: hit_h_omni_low_en + CATDESC: Low energy (6–8 MeV) protons (omnidirectional) + FIELDNAM: Protons 6–8 MeV (omni) + LABLAXIS: H 6–8 MeV (omni) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 hit_h_omni_med_en: <<: *default_uint32 - CATDESC: Medium energy (12–15 MeV) protons (Omnidirectional) - FIELDNAM: Medium energy (12–15 MeV) protons (Omnidirectional) - LABLAXIS: hit_h_omni_med_en + CATDESC: Medium energy (12–15 MeV) protons (omnidirectional) + FIELDNAM: Protons 12–15 MeV (omni) + LABLAXIS: H 12–15 MeV (omni) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -262,8 +273,8 @@ hit_h_omni_med_en: hit_h_a_side_high_en: <<: *default_uint32 CATDESC: High energy (>70 MeV) protons (A-side) - FIELDNAM: High energy (>70 MeV) protons (A-side) - LABLAXIS: hit_h_a_side_high_en + FIELDNAM: Protons >70 MeV (A-side) + LABLAXIS: H >70 MeV (A-side) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -271,35 +282,35 @@ hit_h_a_side_high_en: hit_h_b_side_high_en: <<: *default_uint32 CATDESC: High energy (>70 MeV) protons (B-side) - FIELDNAM: High energy (>70 MeV) protons (B-side) - LABLAXIS: hit_h_b_side_high_en + FIELDNAM: Protons >70 MeV (B-side) + LABLAXIS: H >70 MeV (B-side) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 hit_he_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6–8 MeV/nuc) He (Omnidirectional) - FIELDNAM: Low energy (6–8 MeV/nuc) He (Omnidirectional) - LABLAXIS: hit_he_omni_low_en + CATDESC: Low energy (6–8 MeV/nuc) helium (omnidirectional) + FIELDNAM: Helium 6–8 MeV/nuc (omni) + LABLAXIS: He 6–8 MeV/nuc (omni) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 hit_he_omni_high_en: <<: *default_uint32 - CATDESC: High energy (15–70 MeV/nuc) He (Omnidirectional) - FIELDNAM: High energy (15–70 MeV/nuc) He (Omnidirectional) - LABLAXIS: hit_he_omni_high_en + CATDESC: High energy (15–70 MeV/nuc) helium (omnidirectional) + FIELDNAM: Helium 15–70 MeV/nuc (omni) + LABLAXIS: He 15–70 MeV/nuc (omni) UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 mag_epoch: <<: *default_float32 - CATDESC: MAG instrument epoch - FIELDNAM: mag_epoch - LABLAXIS: mag_epoch + CATDESC: MAG instrument epoch time tag + FIELDNAM: MAG epoch + LABLAXIS: MAG epoch UNITS: ns VALIDMIN: 0.0 VALIDMAX: 9223372036854775807 @@ -307,7 +318,8 @@ mag_epoch: mag_B_GSE: <<: *default_float32 CATDESC: Magnetic field vector in GSE coordinates - FIELDNAM: mag_B_GSE + FIELDNAM: Magnetic field in GSE + LABLAXIS: B (GSE) LABL_PTR_1: component UNITS: nT DEPEND_1: component @@ -317,7 +329,8 @@ mag_B_GSE: mag_B_GSM: <<: *default_float32 CATDESC: Magnetic field vector in GSM coordinates - FIELDNAM: mag_B_GSM + FIELDNAM: Magnetic field in GSM + LABLAXIS: B (GSM) LABL_PTR_1: component UNITS: nT DEPEND_1: component @@ -327,7 +340,8 @@ mag_B_GSM: mag_B_RTN: <<: *default_float32 CATDESC: Magnetic field vector in RTN coordinates - FIELDNAM: mag_B_RTN + FIELDNAM: Magnetic field in RTN + LABLAXIS: B (RTN) LABL_PTR_1: RTN_component UNITS: nT DEPEND_1: RTN_component @@ -337,8 +351,8 @@ mag_B_RTN: mag_B_magnitude: <<: *default_float32 CATDESC: Magnitude of the magnetic field vector - FIELDNAM: mag_B_magnitude - LABLAXIS: "|B|" + FIELDNAM: Magnetic field magnitude + LABLAXIS: B Magnitude UNITS: nT VALIDMIN: 0 VALIDMAX: 65000 @@ -346,8 +360,8 @@ mag_B_magnitude: mag_phi_B_GSM: <<: *default_float32 CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates - FIELDNAM: mag_phi_B_GSM - LABLAXIS: mag_phi_B_GSM + FIELDNAM: B azimuth angle in GSM + LABLAXIS: B azimuth (GSM) UNITS: degrees VALIDMIN: 0 VALIDMAX: 360 @@ -355,8 +369,8 @@ mag_phi_B_GSM: mag_theta_B_GSM: <<: *default_float32 CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates - FIELDNAM: mag_theta_B_GSM - LABLAXIS: mag_theta_B_GSM + FIELDNAM: B elevation angle in GSM + LABLAXIS: B elevation (GSM) UNITS: degrees VALIDMIN: -90 VALIDMAX: 90 @@ -364,8 +378,8 @@ mag_theta_B_GSM: mag_phi_B_GSE: <<: *default_float32 CATDESC: Azimuth angle (φ) of the magnetic field in GSE coordinates - FIELDNAM: mag_phi_B_GSE - LABLAXIS: mag_phi_B_GSE + FIELDNAM: B azimuth angle in GSE + LABLAXIS: B azimuth (GSE) UNITS: degrees VALIDMIN: 0 VALIDMAX: 360 @@ -373,16 +387,17 @@ mag_phi_B_GSE: mag_theta_B_GSE: <<: *default_float32 CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates - FIELDNAM: mag_theta_B_GSE - LABLAXIS: mag_theta_B_GSE + FIELDNAM: B elevation angle in GSE + LABLAXIS: B elevation (GSE) UNITS: degrees VALIDMIN: -90 VALIDMAX: 90 sc_position_GSM: <<: *default_float32 - CATDESC: Spacecraft position in GSM coordinates - FIELDNAM: sc_position_GSM + CATDESC: Spacecraft position vector in GSM coordinates + FIELDNAM: Spacecraft position in GSM + LABLAXIS: SC position (GSM) UNITS: km DEPEND_1: component VALIDMIN: -3000000.0 @@ -390,8 +405,9 @@ sc_position_GSM: sc_velocity_GSM: <<: *default_float32 - CATDESC: Spacecraft velocity in GSM coordinates - FIELDNAM: sc_velocity_GSM + CATDESC: Spacecraft velocity vector in GSM coordinates + FIELDNAM: Spacecraft velocity in GSM + LABLAXIS: SC velocity (GSM) UNITS: km/s DEPEND_1: component VALIDMIN: -1000.0 @@ -399,8 +415,9 @@ sc_velocity_GSM: sc_position_GSE: <<: *default_float32 - CATDESC: Spacecraft position in GSE coordinates - FIELDNAM: sc_position_GSE + CATDESC: Spacecraft position vector in GSE coordinates + FIELDNAM: Spacecraft position in GSE + LABLAXIS: SC position (GSE) UNITS: km DEPEND_1: component VALIDMIN: -3000000.0 @@ -408,8 +425,9 @@ sc_position_GSE: sc_velocity_GSE: <<: *default_float32 - CATDESC: Spacecraft velocity in GSE coordinates - FIELDNAM: sc_velocity_GSE + CATDESC: Spacecraft velocity vector in GSE coordinates + FIELDNAM: Spacecraft velocity in GSE + LABLAXIS: SC velocity (GSE) UNITS: km/s DEPEND_1: component VALIDMIN: -1000.0 @@ -418,8 +436,8 @@ sc_velocity_GSE: swapi_pseudo_proton_density: <<: *default_float32 CATDESC: Pseudo density of solar wind protons - FIELDNAM: swapi_pseudo_proton_density - LABLAXIS: swapi_pseudo_proton_density + FIELDNAM: SW proton pseudo density + LABLAXIS: proton density UNITS: 1/cm^3 VALIDMIN: 0.0 VALIDMAX: 1000.0 @@ -427,8 +445,8 @@ swapi_pseudo_proton_density: swapi_pseudo_proton_speed: <<: *default_float32 CATDESC: Pseudo speed of solar wind protons in solar inertial frame - FIELDNAM: swapi_pseudo_proton_speed - LABLAXIS: swapi_pseudo_proton_speed + FIELDNAM: SW proton pseudo speed + LABLAXIS: proton speed UNITS: km/sec VALIDMIN: 0.0 VALIDMAX: 10000.0 @@ -436,8 +454,8 @@ swapi_pseudo_proton_speed: swapi_pseudo_proton_temperature: <<: *default_float32 CATDESC: Pseudo temperature of solar wind protons in plasma frame - FIELDNAM: swapi_pseudo_proton_temperature - LABLAXIS: swapi_pseudo_proton_temperature + FIELDNAM: SW proton pseudo temperature + LABLAXIS: proton temperature UNITS: Kelvin VALIDMIN: 0.0 VALIDMAX: 50000000.0 @@ -446,8 +464,8 @@ swapi_pseudo_proton_temperature: swe_normalized_counts: <<: *default_uint32 CATDESC: Normalized electron counts - FIELDNAM: swe_normalized_counts - LABLAXIS: swe_normalized_counts + FIELDNAM: SWE normalized counts + LABLAXIS: SWE norm counts UNITS: Normalized counts VALIDMIN: 0 VALIDMAX: 500000 @@ -455,9 +473,9 @@ swe_normalized_counts: swe_counterstreaming_electrons: <<: *default_uint8 - CATDESC: Counterstreaming electrons - FIELDNAM: swe_counterstreaming_electrons - LABLAXIS: swe_counterstreaming_electrons - UNITS: "" + CATDESC: Counterstreaming electrons flag (0=no, 1=yes) + FIELDNAM: Counterstreaming electrons + LABLAXIS: Counterstreaming e- + UNITS: " " VALIDMIN: 0 VALIDMAX: 1 diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index d4e7db7afd..5cc1aeb9a4 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -61,21 +61,21 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("esa_step", check_schema=False), ) - energy_ranges = xr.DataArray( + energy_range = xr.DataArray( data=np.arange(15, dtype=np.uint8), - name="codice_hi_h_energy_ranges", - dims=["codice_hi_h_energy_ranges"], + name="codice_hi_h_energy_range", + dims=["codice_hi_h_energy_range"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_energy_ranges", check_schema=False + "codice_hi_h_energy_range", check_schema=False ), ) - elevation = xr.DataArray( + azimuth = xr.DataArray( data=np.arange(4, dtype=np.uint8), - name="codice_hi_h_elevation", - dims=["codice_hi_h_elevation"], + name="codice_hi_h_azimuth", + dims=["codice_hi_h_azimuth"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_elevation", check_schema=False + "codice_hi_h_azimuth", check_schema=False ), ) @@ -88,14 +88,24 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) + spin_sector = xr.DataArray( + data=np.arange(4, dtype=np.uint8), + name="codice_hi_h_spin_sector", + dims=["codice_hi_h_spin_sector"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_h_spin_sector", check_schema=False + ), + ) + coords = { "epoch": epoch, "component": component, "RTN_component": rtn_component, "esa_step": esa_step, - "codice_hi_h_energy_ranges": energy_ranges, - "codice_hi_h_elevation": elevation, "codice_hi_h_spin_angle": spin_angle, + "codice_hi_h_energy_range": energy_range, + "codice_hi_h_spin_sector": spin_sector, + "codice_hi_h_azimuth": azimuth, } dataset = xr.Dataset( coords=coords, @@ -115,12 +125,13 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dims = ["epoch", "RTN_component"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("codice_hi"): - data = np.full((n, 15, 4, 4), fillval, dtype=np.float32) + data = np.full((n, 4, 15, 4, 4), fillval, dtype=np.float32) dims = [ "epoch", - "codice_hi_h_energy_ranges", - "codice_hi_h_elevation", "codice_hi_h_spin_angle", + "codice_hi_h_energy_range", + "codice_hi_h_spin_sector", + "codice_hi_h_azimuth", ] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "swe_counterstreaming_electrons": @@ -149,10 +160,6 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "met_in_utc", "ttj2000ns", "last_modified", - "sc_position_GSM", - "sc_position_GSE", - "sc_velocity_GSM", - "sc_velocity_GSE", "mag_hk_status", "spice_kernels", "instrument", @@ -163,7 +170,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 elif key.startswith("swe_normalized_counts"): dataset[key].data[i, :] = val elif key.startswith("codice_hi"): - dataset[key].data[i, :, :, :] = val + dataset[key].data[i, :, :, :, :] = val else: dataset[key].data[i] = val From efb5a9fd6576a1cd0c6d6f1690518782b17bc853 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 13:05:30 -0700 Subject: [PATCH 04/42] updated var notes --- .../config/imap_ialirt_l1_variable_attrs.yaml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 6dc7dfe06f..458e9c9eea 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -143,6 +143,7 @@ codice_hi_h: DEPEND_2: codice_hi_h_energy_range DEPEND_3: codice_hi_h_spin_sector DEPEND_4: codice_hi_h_azimuth + VAR_NOTES: Energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. codice_lo_c_over_o_abundance: <<: *default_float32 @@ -470,12 +471,14 @@ swe_normalized_counts: VALIDMIN: 0 VALIDMAX: 500000 DEPEND_1: esa_step + VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. swe_counterstreaming_electrons: - <<: *default_uint8 - CATDESC: Counterstreaming electrons flag (0=no, 1=yes) - FIELDNAM: Counterstreaming electrons - LABLAXIS: Counterstreaming e- - UNITS: " " - VALIDMIN: 0 - VALIDMAX: 1 + <<: *default_uint8 + CATDESC: Counterstreaming electrons flag (0=no, 1=yes) + FIELDNAM: Counterstreaming electrons + LABLAXIS: Counterstreaming e- + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 1 + VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow.. From d216133fad5953b6689f24855f6e4c2ff4c7c626 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 13:31:12 -0700 Subject: [PATCH 05/42] add varnotes --- .../config/imap_ialirt_l1_variable_attrs.yaml | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 458e9c9eea..33cc618fe1 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -207,6 +207,7 @@ hit_e_a_side_low_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_e_a_side_med_en: <<: *default_uint32 @@ -216,6 +217,7 @@ hit_e_a_side_med_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_e_a_side_high_en: <<: *default_uint32 @@ -225,6 +227,7 @@ hit_e_a_side_high_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_e_b_side_low_en: <<: *default_uint32 @@ -234,6 +237,7 @@ hit_e_b_side_low_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_e_b_side_med_en: <<: *default_uint32 @@ -243,6 +247,7 @@ hit_e_b_side_med_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_e_b_side_high_en: <<: *default_uint32 @@ -252,6 +257,7 @@ hit_e_b_side_high_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_h_omni_low_en: <<: *default_uint32 @@ -279,6 +285,7 @@ hit_h_a_side_high_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_h_b_side_high_en: <<: *default_uint32 @@ -288,6 +295,7 @@ hit_h_b_side_high_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: A-side indicates the HIT aperture that is nominally looking in the anti-sunward direction. B-side indicates the aperture looking nominally in the sunward direction. hit_he_omni_low_en: <<: *default_uint32 @@ -314,7 +322,7 @@ mag_epoch: LABLAXIS: MAG epoch UNITS: ns VALIDMIN: 0.0 - VALIDMAX: 9223372036854775807 + VALIDMAX: 9223372036854775807.0 mag_B_GSE: <<: *default_float32 @@ -324,8 +332,8 @@ mag_B_GSE: LABL_PTR_1: component UNITS: nT DEPEND_1: component - VALIDMIN: -65000 - VALIDMAX: 65000 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 mag_B_GSM: <<: *default_float32 @@ -335,8 +343,8 @@ mag_B_GSM: LABL_PTR_1: component UNITS: nT DEPEND_1: component - VALIDMIN: -65000 - VALIDMAX: 65000 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 mag_B_RTN: <<: *default_float32 @@ -346,8 +354,8 @@ mag_B_RTN: LABL_PTR_1: RTN_component UNITS: nT DEPEND_1: RTN_component - VALIDMIN: -65000 - VALIDMAX: 65000 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 mag_B_magnitude: <<: *default_float32 @@ -355,8 +363,8 @@ mag_B_magnitude: FIELDNAM: Magnetic field magnitude LABLAXIS: B Magnitude UNITS: nT - VALIDMIN: 0 - VALIDMAX: 65000 + VALIDMIN: 0.0 + VALIDMAX: 65000.0 mag_phi_B_GSM: <<: *default_float32 @@ -364,8 +372,8 @@ mag_phi_B_GSM: FIELDNAM: B azimuth angle in GSM LABLAXIS: B azimuth (GSM) UNITS: degrees - VALIDMIN: 0 - VALIDMAX: 360 + VALIDMIN: 0.0 + VALIDMAX: 360.0 mag_theta_B_GSM: <<: *default_float32 @@ -373,8 +381,8 @@ mag_theta_B_GSM: FIELDNAM: B elevation angle in GSM LABLAXIS: B elevation (GSM) UNITS: degrees - VALIDMIN: -90 - VALIDMAX: 90 + VALIDMIN: -90.0 + VALIDMAX: 90.0 mag_phi_B_GSE: <<: *default_float32 @@ -382,8 +390,8 @@ mag_phi_B_GSE: FIELDNAM: B azimuth angle in GSE LABLAXIS: B azimuth (GSE) UNITS: degrees - VALIDMIN: 0 - VALIDMAX: 360 + VALIDMIN: 0.0 + VALIDMAX: 360.0 mag_theta_B_GSE: <<: *default_float32 @@ -391,8 +399,8 @@ mag_theta_B_GSE: FIELDNAM: B elevation angle in GSE LABLAXIS: B elevation (GSE) UNITS: degrees - VALIDMIN: -90 - VALIDMAX: 90 + VALIDMIN: -90.0 + VALIDMAX: 90.0 sc_position_GSM: <<: *default_float32 @@ -481,4 +489,4 @@ swe_counterstreaming_electrons: UNITS: " " VALIDMIN: 0 VALIDMAX: 1 - VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow.. + VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow. From 80a887b9df1135dde13234551274591ee71163c6 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 13:48:15 -0700 Subject: [PATCH 06/42] update to hit description --- .../config/imap_ialirt_global_cdf_attrs.yaml | 4 +-- .../config/imap_ialirt_l1_variable_attrs.yaml | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml index 0b300fa93d..69639fcac0 100644 --- a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml @@ -9,10 +9,10 @@ instrument_base: &instrument_base HIT (High-energy Ion Telescope), MAG (Magnetometer), and SWAPI (Solar Wind and Pickup Ion). See https://imap.princeton.edu for more details. - Instrument_type: "Multiple Instruments" + Instrument_type: CoDICE - Particles (space), HIT - Particles (space), MAG - Magnetic Fields (space), SWAPI - Particles (space), SWE - Particles (space) imap_ialirt_l1_realtime: <<: *instrument_base Data_type: L1_REALTIME>Level-1 Realtime Logical_source: imap_ialirt_l1_realtime - Logical_source_description: IMAP-IALiRT Instrument Level-1 Realtime Data. + Logical_source_description: IMAP Active Link for Real-Time (I-ALiRT) Level-1 Realtime Data. diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 33cc618fe1..aaf99ac14b 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -203,7 +203,7 @@ hit_e_a_side_low_en: <<: *default_uint32 CATDESC: Low energy (>0.5 MeV) electrons (A-side) FIELDNAM: Electrons low energy (A-side) - LABLAXIS: e- low energy (A) + LABLAXIS: A-side e- >0.5 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -213,7 +213,7 @@ hit_e_a_side_med_en: <<: *default_uint32 CATDESC: Medium energy (<1 MeV) electrons (A-side) FIELDNAM: Electrons medium energy (A-side) - LABLAXIS: e- medium energy (A) + LABLAXIS: A-side e- <1 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -223,7 +223,7 @@ hit_e_a_side_high_en: <<: *default_uint32 CATDESC: High energy (>3 MeV) electrons (A-side) FIELDNAM: Electrons high energy (A-side) - LABLAXIS: e- high energy (A) + LABLAXIS: A-side e- >3 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -233,7 +233,7 @@ hit_e_b_side_low_en: <<: *default_uint32 CATDESC: Low energy (>0.5 MeV) electrons (B-side) FIELDNAM: Electrons low energy (B-side) - LABLAXIS: e- low energy (B) + LABLAXIS: B-side e- >0.5 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -242,8 +242,8 @@ hit_e_b_side_low_en: hit_e_b_side_med_en: <<: *default_uint32 CATDESC: Medium energy (<1 MeV) electrons (B-side) - FIELDNAM: Electrons medium energy (B-side) - LABLAXIS: e- medium energy (B) + FIELDNAM: Electrons medium energy + LABLAXIS: B-side e- <1 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -253,7 +253,7 @@ hit_e_b_side_high_en: <<: *default_uint32 CATDESC: High energy (>3 MeV) electrons (B-side) FIELDNAM: Electrons high energy (B-side) - LABLAXIS: e- high energy (B) + LABLAXIS: B-side e- >3 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -263,7 +263,7 @@ hit_h_omni_low_en: <<: *default_uint32 CATDESC: Low energy (6–8 MeV) protons (omnidirectional) FIELDNAM: Protons 6–8 MeV (omni) - LABLAXIS: H 6–8 MeV (omni) + LABLAXIS: Omni H 6–8 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -272,7 +272,7 @@ hit_h_omni_med_en: <<: *default_uint32 CATDESC: Medium energy (12–15 MeV) protons (omnidirectional) FIELDNAM: Protons 12–15 MeV (omni) - LABLAXIS: H 12–15 MeV (omni) + LABLAXIS: Omni H 12–15 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -281,7 +281,7 @@ hit_h_a_side_high_en: <<: *default_uint32 CATDESC: High energy (>70 MeV) protons (A-side) FIELDNAM: Protons >70 MeV (A-side) - LABLAXIS: H >70 MeV (A-side) + LABLAXIS: A-side H >70 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -291,7 +291,7 @@ hit_h_b_side_high_en: <<: *default_uint32 CATDESC: High energy (>70 MeV) protons (B-side) FIELDNAM: Protons >70 MeV (B-side) - LABLAXIS: H >70 MeV (B-side) + LABLAXIS: B-side H >70 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -301,7 +301,7 @@ hit_he_omni_low_en: <<: *default_uint32 CATDESC: Low energy (6–8 MeV/nuc) helium (omnidirectional) FIELDNAM: Helium 6–8 MeV/nuc (omni) - LABLAXIS: He 6–8 MeV/nuc (omni) + LABLAXIS: Omni He 6–8 MeV/nuc UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -310,7 +310,7 @@ hit_he_omni_high_en: <<: *default_uint32 CATDESC: High energy (15–70 MeV/nuc) helium (omnidirectional) FIELDNAM: Helium 15–70 MeV/nuc (omni) - LABLAXIS: He 15–70 MeV/nuc (omni) + LABLAXIS: Omni He 15–70 MeV/nuc UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 From c47f266b62b3401b2a88bc5109f216eeb7942f49 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 14:44:44 -0700 Subject: [PATCH 07/42] mag and swapi corrections --- .../config/imap_ialirt_l1_variable_attrs.yaml | 78 +++++++++++-------- imap_processing/ialirt/utils/constants.py | 8 ++ imap_processing/ialirt/utils/create_xarray.py | 41 ++++++---- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index aaf99ac14b..b11b2b0656 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -9,19 +9,24 @@ default_attrs: &default VAR_TYPE: data UNITS: " " -component: - CATDESC: Component direction (x, y, z) - FIELDNAM: Vector component - LABLAXIS: Component - FORMAT: A3 +B_GSM_labels: + CATDESC: B GSM component labels + FIELDNAM: B GSM component labels + FORMAT: A8 VAR_TYPE: metadata UNITS: " " -RTN_component: - CATDESC: Component direction (radial, tangential, normal) - FIELDNAM: RTN vector component - LABLAXIS: RTN component - FORMAT: A3 +B_GSE_labels: + CATDESC: B GSE component labels + FIELDNAM: B GSE component labels + FORMAT: A8 + VAR_TYPE: metadata + UNITS: " " + +B_RTN_labels: + CATDESC: B RTN component labels B radial (RTN), B tangential (RTN), B normal (RTN) + FIELDNAM: B RTN component labels + FORMAT: A18 VAR_TYPE: metadata UNITS: " " @@ -267,6 +272,7 @@ hit_h_omni_low_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. hit_h_omni_med_en: <<: *default_uint32 @@ -276,6 +282,7 @@ hit_h_omni_med_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. hit_h_a_side_high_en: <<: *default_uint32 @@ -305,6 +312,7 @@ hit_he_omni_low_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. hit_he_omni_high_en: <<: *default_uint32 @@ -314,6 +322,7 @@ hit_he_omni_high_en: UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 + VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. mag_epoch: <<: *default_float32 @@ -327,40 +336,40 @@ mag_epoch: mag_B_GSE: <<: *default_float32 CATDESC: Magnetic field vector in GSE coordinates - FIELDNAM: Magnetic field in GSE + FIELDNAM: B vector in GSE coordinates LABLAXIS: B (GSE) - LABL_PTR_1: component + LABL_PTR_1: B_GSE_labels UNITS: nT - DEPEND_1: component + FORMAT: F13.6 VALIDMIN: -65000.0 VALIDMAX: 65000.0 mag_B_GSM: <<: *default_float32 CATDESC: Magnetic field vector in GSM coordinates - FIELDNAM: Magnetic field in GSM + FIELDNAM: B vector in GSM coordinates LABLAXIS: B (GSM) - LABL_PTR_1: component + LABL_PTR_1: B_GSM_labels UNITS: nT - DEPEND_1: component + FORMAT: F13.6 VALIDMIN: -65000.0 VALIDMAX: 65000.0 mag_B_RTN: <<: *default_float32 CATDESC: Magnetic field vector in RTN coordinates - FIELDNAM: Magnetic field in RTN + FIELDNAM: B vector in RTN coordinates LABLAXIS: B (RTN) - LABL_PTR_1: RTN_component + LABL_PTR_1: B_RTN_labels UNITS: nT - DEPEND_1: RTN_component + FORMAT: F13.6 VALIDMIN: -65000.0 VALIDMAX: 65000.0 mag_B_magnitude: <<: *default_float32 CATDESC: Magnitude of the magnetic field vector - FIELDNAM: Magnetic field magnitude + FIELDNAM: B vector magnitude LABLAXIS: B Magnitude UNITS: nT VALIDMIN: 0.0 @@ -369,7 +378,7 @@ mag_B_magnitude: mag_phi_B_GSM: <<: *default_float32 CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates - FIELDNAM: B azimuth angle in GSM + FIELDNAM: B azimuth angle in GSM coordinates LABLAXIS: B azimuth (GSM) UNITS: degrees VALIDMIN: 0.0 @@ -378,7 +387,7 @@ mag_phi_B_GSM: mag_theta_B_GSM: <<: *default_float32 CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates - FIELDNAM: B elevation angle in GSM + FIELDNAM: B elevation angle in GSM coordinates LABLAXIS: B elevation (GSM) UNITS: degrees VALIDMIN: -90.0 @@ -387,7 +396,7 @@ mag_theta_B_GSM: mag_phi_B_GSE: <<: *default_float32 CATDESC: Azimuth angle (φ) of the magnetic field in GSE coordinates - FIELDNAM: B azimuth angle in GSE + FIELDNAM: B azimuth angle in GSE coordinates LABLAXIS: B azimuth (GSE) UNITS: degrees VALIDMIN: 0.0 @@ -396,7 +405,7 @@ mag_phi_B_GSE: mag_theta_B_GSE: <<: *default_float32 CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates - FIELDNAM: B elevation angle in GSE + FIELDNAM: B elevation angle in GSE coordinates LABLAXIS: B elevation (GSE) UNITS: degrees VALIDMIN: -90.0 @@ -405,10 +414,10 @@ mag_theta_B_GSE: sc_position_GSM: <<: *default_float32 CATDESC: Spacecraft position vector in GSM coordinates - FIELDNAM: Spacecraft position in GSM + FIELDNAM: Spacecraft position in GSM coordinates LABLAXIS: SC position (GSM) UNITS: km - DEPEND_1: component + LABL_PTR_1: B_GSM_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 @@ -418,7 +427,7 @@ sc_velocity_GSM: FIELDNAM: Spacecraft velocity in GSM LABLAXIS: SC velocity (GSM) UNITS: km/s - DEPEND_1: component + LABL_PTR_1: B_GSM_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 @@ -428,7 +437,7 @@ sc_position_GSE: FIELDNAM: Spacecraft position in GSE LABLAXIS: SC position (GSE) UNITS: km - DEPEND_1: component + LABL_PTR_1: B_GSE_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 @@ -438,7 +447,7 @@ sc_velocity_GSE: FIELDNAM: Spacecraft velocity in GSE LABLAXIS: SC velocity (GSE) UNITS: km/s - DEPEND_1: component + LABL_PTR_1: B_GSE_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 @@ -446,28 +455,31 @@ swapi_pseudo_proton_density: <<: *default_float32 CATDESC: Pseudo density of solar wind protons FIELDNAM: SW proton pseudo density - LABLAXIS: proton density + LABLAXIS: SW p pseudo N UNITS: 1/cm^3 VALIDMIN: 0.0 VALIDMAX: 1000.0 + VAR_NOTES: The pseudo density is derived from the SWAPI proton moments but does not account for instrument calibration factors. swapi_pseudo_proton_speed: <<: *default_float32 CATDESC: Pseudo speed of solar wind protons in solar inertial frame - FIELDNAM: SW proton pseudo speed - LABLAXIS: proton speed + FIELDNAM: SW proton pseudo V in solar inertial frame + LABLAXIS: SW p pseudo V UNITS: km/sec VALIDMIN: 0.0 VALIDMAX: 10000.0 + VAR_NOTES: The pseudo speed is derived from the SWAPI proton moments but does not account for instrument calibration factors. swapi_pseudo_proton_temperature: <<: *default_float32 CATDESC: Pseudo temperature of solar wind protons in plasma frame FIELDNAM: SW proton pseudo temperature - LABLAXIS: proton temperature + LABLAXIS: SW p pseudo T UNITS: Kelvin VALIDMIN: 0.0 VALIDMAX: 50000000.0 + VAR_NOTES: The pseudo temperature is derived from the SWAPI proton moments but does not account for instrument calibration factors. # SWE Normalized Counts swe_normalized_counts: diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 934ffe1e13..2f73f0055c 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -59,4 +59,12 @@ "swe_normalized_counts", # SWE Counterstreaming flag "swe_counterstreaming_electrons", + # Spacecraft position in GSM coordinates + "sc_position_GSM", + # Spacecraft velocity in GSM coordinates + "sc_velocity_GSM", + # Spacecraft position in GSE coordinates + "sc_position_GSE", + # Spacecraft velocity in GSE coordinates + "sc_velocity_GSE", ] diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 5cc1aeb9a4..5700b499e4 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -40,18 +40,26 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dims=["epoch"], attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False), ) - component = xr.DataArray( - ["x", "y", "z"], - name="component", - dims=["component"], - attrs=cdf_manager.get_variable_attributes("component", check_schema=False), + + gsm_component = xr.DataArray( + ["Bx (GSM)", "By (GSM)", "Bz (GSM)"], + name="B_GSM_labels", + dims=["B_GSM_labels"], + attrs=cdf_manager.get_variable_attributes("B_GSM_labels", check_schema=False), + ) + + gse_component = xr.DataArray( + ["Bx (GSE)", "By (GSE)", "Bz (GSE)"], + name="B_GSE_labels", + dims=["B_GSE_labels"], + attrs=cdf_manager.get_variable_attributes("B_GSE_labels", check_schema=False), ) rtn_component = xr.DataArray( - ["radial", "tangential", "normal"], - name="RTN_component", - dims=["RTN_component"], - attrs=cdf_manager.get_variable_attributes("RTN_component", check_schema=False), + ["B radial (RTN)", "B tangential (RTN)", "B normal (RTN)"], + name="B_RTN_labels", + dims=["B_RTN_labels"], + attrs=cdf_manager.get_variable_attributes("B_RTN_labels", check_schema=False), ) esa_step = xr.DataArray( @@ -99,8 +107,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 coords = { "epoch": epoch, - "component": component, - "RTN_component": rtn_component, + "B_GSM_labels": gsm_component, + "B_GSE_labels": gse_component, + "B_RTN_labels": rtn_component, "esa_step": esa_step, "codice_hi_h_spin_angle": spin_angle, "codice_hi_h_energy_range": energy_range, @@ -116,13 +125,17 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for key in IALIRT_KEYS: attrs = cdf_manager.get_variable_attributes(key, check_schema=False) fillval = attrs.get("FILLVAL") - if key in ["mag_B_GSE", "mag_B_GSM"]: + if key in ["mag_B_GSE", "sc_position_GSE", "sc_velocity_GSE"]: + data = np.full((n, 3), fillval, dtype=np.float32) + dims = ["epoch", "B_GSE_labels"] + dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) + elif key in ["mag_B_GSM", "sc_position_GSM", "sc_velocity_GSM"]: data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "component"] + dims = ["epoch", "B_GSM_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "mag_B_RTN": data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "RTN_component"] + dims = ["epoch", "B_RTN_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("codice_hi"): data = np.full((n, 4, 15, 4, 4), fillval, dtype=np.float32) From cb5e234663c34e15d8686f8e887619f7a4ab17d2 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 15:26:00 -0700 Subject: [PATCH 08/42] update for swe --- .../config/imap_ialirt_l1_variable_attrs.yaml | 18 ++++++++----- imap_processing/ialirt/utils/create_xarray.py | 25 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index b11b2b0656..fc48ebd1d9 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -40,6 +40,13 @@ esa_step: VAR_TYPE: metadata UNITS: " " +swe_electron_energy_labels: + CATDESC: SWE electron channel labels (nominal central energies) + FIELDNAM: SWE electron channel labels + FORMAT: A12 + VAR_TYPE: support_data + UNITS: " " + codice_hi_h_spin_angle: CATDESC: CoDICE-Hi spin angle bin FIELDNAM: CoDICE-Hi spin angle bin @@ -485,20 +492,19 @@ swapi_pseudo_proton_temperature: swe_normalized_counts: <<: *default_uint32 CATDESC: Normalized electron counts - FIELDNAM: SWE normalized counts - LABLAXIS: SWE norm counts + FIELDNAM: Normalized electron counts UNITS: Normalized counts VALIDMIN: 0 VALIDMAX: 500000 - DEPEND_1: esa_step + LABL_PTR_1: swe_electron_energy_labels VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. swe_counterstreaming_electrons: <<: *default_uint8 - CATDESC: Counterstreaming electrons flag (0=no, 1=yes) - FIELDNAM: Counterstreaming electrons + CATDESC: Counterstreaming electrons flag (1=counterstreaming, 0=unidirectional) + FIELDNAM: Counterstreaming e- (1=counterstreaming) LABLAXIS: Counterstreaming e- - UNITS: " " + UNITS: "1=counterstreaming, 0=unidirectional" VALIDMIN: 0 VALIDMAX: 1 VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow. diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 5700b499e4..ad467128b6 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -62,11 +62,22 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("B_RTN_labels", check_schema=False), ) - esa_step = xr.DataArray( - data=np.arange(8, dtype=np.uint8), - name="esa_step", - dims=["esa_step"], - attrs=cdf_manager.get_variable_attributes("esa_step", check_schema=False), + swe_electron_energy_labels = xr.DataArray( + [ + "100.4 eV", + "140 eV", + "194 eV", + "270 eV", + "376 eV", + "523 eV", + "727 eV", + "1011 eV", + ], + name="swe_electron_energy_labels", + dims=["swe_electron_energy_labels"], + attrs=cdf_manager.get_variable_attributes( + "swe_electron_energy_labels", check_schema=False + ), ) energy_range = xr.DataArray( @@ -110,7 +121,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, - "esa_step": esa_step, + "swe_electron_energy_labels": swe_electron_energy_labels, "codice_hi_h_spin_angle": spin_angle, "codice_hi_h_energy_range": energy_range, "codice_hi_h_spin_sector": spin_sector, @@ -153,7 +164,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("swe"): data = np.full((n, 8), fillval, dtype=np.uint32) - dims = ["epoch", "esa_step"] + dims = ["epoch", "swe_electron_energy_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("hit"): data = np.full(n, fillval, dtype=np.uint32) From d3521999da56cdcd2d352f6b8363e41c32a06658 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 15:39:00 -0700 Subject: [PATCH 09/42] update spacecraft labels --- .../config/imap_ialirt_l1_variable_attrs.yaml | 28 ++++++++++++++----- imap_processing/ialirt/utils/create_xarray.py | 28 +++++++++++++++++-- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index fc48ebd1d9..688dcb23f8 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -13,21 +13,35 @@ B_GSM_labels: CATDESC: B GSM component labels FIELDNAM: B GSM component labels FORMAT: A8 - VAR_TYPE: metadata + VAR_TYPE: support_data UNITS: " " B_GSE_labels: CATDESC: B GSE component labels FIELDNAM: B GSE component labels FORMAT: A8 - VAR_TYPE: metadata + VAR_TYPE: support_data UNITS: " " B_RTN_labels: CATDESC: B RTN component labels B radial (RTN), B tangential (RTN), B normal (RTN) FIELDNAM: B RTN component labels FORMAT: A18 - VAR_TYPE: metadata + VAR_TYPE: support_data + UNITS: " " + +sc_GSM_labels: + CATDESC: Spacecraft position and velocity component labels in GSM coordinates + FIELDNAM: Spacecraft GSM component labels + FORMAT: A8 + VAR_TYPE: support_data + UNITS: " " + +sc_GSE_labels: + CATDESC: Spacecraft position and velocity component labels in GSE coordinates + FIELDNAM: Spacecraft GSE component labels + FORMAT: A8 + VAR_TYPE: support_data UNITS: " " esa_step: @@ -424,7 +438,7 @@ sc_position_GSM: FIELDNAM: Spacecraft position in GSM coordinates LABLAXIS: SC position (GSM) UNITS: km - LABL_PTR_1: B_GSM_labels + LABL_PTR_1: sc_GSM_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 @@ -434,7 +448,7 @@ sc_velocity_GSM: FIELDNAM: Spacecraft velocity in GSM LABLAXIS: SC velocity (GSM) UNITS: km/s - LABL_PTR_1: B_GSM_labels + LABL_PTR_1: sc_GSM_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 @@ -444,7 +458,7 @@ sc_position_GSE: FIELDNAM: Spacecraft position in GSE LABLAXIS: SC position (GSE) UNITS: km - LABL_PTR_1: B_GSE_labels + LABL_PTR_1: sc_GSE_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 @@ -454,7 +468,7 @@ sc_velocity_GSE: FIELDNAM: Spacecraft velocity in GSE LABLAXIS: SC velocity (GSE) UNITS: km/s - LABL_PTR_1: B_GSE_labels + LABL_PTR_1: sc_GSE_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index ad467128b6..59ad4cfe80 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -41,6 +41,20 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False), ) + sc_gsm_component = xr.DataArray( + ["x (GSM)", "y (GSM)", "z (GSM)"], + name="sc_GSM_labels", + dims=["sc_GSM_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSM_labels", check_schema=False), + ) + + sc_gse_component = xr.DataArray( + ["x (GSE)", "y (GSE)", "z (GSE)"], + name="sc_GSE_labels", + dims=["sc_GSE_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSE_labels", check_schema=False), + ) + gsm_component = xr.DataArray( ["Bx (GSM)", "By (GSM)", "Bz (GSM)"], name="B_GSM_labels", @@ -121,6 +135,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, + "sc_GSM_labels": sc_gsm_component, + "sc_GSE_labels": sc_gse_component, "swe_electron_energy_labels": swe_electron_energy_labels, "codice_hi_h_spin_angle": spin_angle, "codice_hi_h_energy_range": energy_range, @@ -136,11 +152,19 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for key in IALIRT_KEYS: attrs = cdf_manager.get_variable_attributes(key, check_schema=False) fillval = attrs.get("FILLVAL") - if key in ["mag_B_GSE", "sc_position_GSE", "sc_velocity_GSE"]: + if key == "mag_B_GSE": data = np.full((n, 3), fillval, dtype=np.float32) dims = ["epoch", "B_GSE_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key in ["mag_B_GSM", "sc_position_GSM", "sc_velocity_GSM"]: + elif key in ["sc_position_GSE", "sc_velocity_GSE"]: + data = np.full((n, 3), fillval, dtype=np.float32) + dims = ["epoch", "sc_GSE_labels"] + dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) + elif key in ["sc_position_GSM", "sc_velocity_GSM"]: + data = np.full((n, 3), fillval, dtype=np.float32) + dims = ["epoch", "sc_GSM_labels"] + dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) + elif key == "mag_B_GSM": data = np.full((n, 3), fillval, dtype=np.float32) dims = ["epoch", "B_GSM_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) From 9cf9efda11eefa93921ac83b359ac8645e255460 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 15:45:44 -0700 Subject: [PATCH 10/42] add codice lo --- .../config/imap_ialirt_l1_variable_attrs.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 688dcb23f8..7612fef510 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -179,6 +179,7 @@ codice_lo_c_over_o_abundance: UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 codice_lo_mg_over_o_abundance: <<: *default_float32 @@ -188,6 +189,7 @@ codice_lo_mg_over_o_abundance: UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 codice_lo_fe_over_o_abundance: <<: *default_float32 @@ -197,33 +199,37 @@ codice_lo_fe_over_o_abundance: UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 codice_lo_c_plus_6_over_c_plus_5_ratio: <<: *default_float32 CATDESC: C+6/C+5 charge state ratio - FIELDNAM: C6+/C5+ charge ratio - LABLAXIS: C6+/C5+ + FIELDNAM: C+6/C+5 charge state ratio + LABLAXIS: C+6/C+5 ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 codice_lo_o_plus_7_over_o_plus_6_ratio: <<: *default_float32 CATDESC: O+7/O+6 charge state ratio - FIELDNAM: O7+/O6+ charge ratio - LABLAXIS: O7+/O6+ + FIELDNAM: O7+/O6+ charge state ratio + LABLAXIS: O7+/O6+ ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 codice_lo_fe_low_over_fe_high_ratio: <<: *default_float32 CATDESC: Fe low/high charge state ratio (low=6–12, high=13–17) - FIELDNAM: Fe low/high charge ratio - LABLAXIS: Fe low/high + FIELDNAM: Fe low/high charge state ratio + LABLAXIS: Fe low/high ratio UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 + FORMAT: F16.6 hit_e_a_side_low_en: <<: *default_uint32 From 93ea30e00af1bae63bfda18eac3b2d76931f0d4b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 16:35:09 -0700 Subject: [PATCH 11/42] cdf updates to codice --- .../config/imap_ialirt_l1_variable_attrs.yaml | 49 +++++++++---------- imap_processing/ialirt/utils/create_xarray.py | 26 ++++++++-- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 7612fef510..9d8d36a97f 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -65,41 +65,32 @@ codice_hi_h_spin_angle: CATDESC: CoDICE-Hi spin angle bin FIELDNAM: CoDICE-Hi spin angle bin LABLAXIS: Spin angle bin - FORMAT: I3 - VALIDMIN: 0 - VALIDMAX: 3 - VAR_TYPE: metadata - UNITS: " " + FORMAT: A20 + VAR_TYPE: support_data + UNITS: "TODO" codice_hi_h_energy_range: - CATDESC: CoDICE-Hi H energy range - FIELDNAM: CoDICE-Hi H energy range - LABLAXIS: Energy range - FORMAT: I3 - VALIDMIN: 0 - VALIDMAX: 14 - VAR_TYPE: metadata - UNITS: " " + CATDESC: CoDICE-Hi proton energy bin labels (MeV) + FIELDNAM: CoDICE-Hi proton energy bin labels + FORMAT: A20 + VAR_TYPE: support_data + UNITS: "MeV" codice_hi_h_spin_sector: CATDESC: CoDICE-Hi spin sector FIELDNAM: CoDICE-Hi spin sector LABLAXIS: Spin sector - FORMAT: I3 - VALIDMIN: 0 - VALIDMAX: 3 - VAR_TYPE: metadata - UNITS: " " + FORMAT: A20 + VAR_TYPE: support_data + UNITS: "TODO" codice_hi_h_azimuth: CATDESC: CoDICE-Hi azimuth bin FIELDNAM: CoDICE-Hi azimuth bin LABLAXIS: Azimuth bin - FORMAT: I3 - VALIDMIN: 0 - VALIDMAX: 3 - VAR_TYPE: metadata - UNITS: " " + FORMAT: A20 + VAR_TYPE: support_data + UNITS: "TODO" default_uint8_attrs: &default_uint8 <<: *default @@ -159,17 +150,23 @@ default_float64_attrs: &default_float64 codice_hi_h: <<: *default_float32 - CATDESC: H intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins + CATDESC: Proton intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins FIELDNAM: H intensity LABLAXIS: H intensity - UNITS: "# / cm2-sr-s- MeV" + UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 DEPEND_1: codice_hi_h_spin_angle + LABL_PTR_1: codice_hi_h_spin_angle DEPEND_2: codice_hi_h_energy_range + LABL_PTR_2: codice_hi_h_energy_range DEPEND_3: codice_hi_h_spin_sector + LABL_PTR_3: codice_hi_h_spin_sector DEPEND_4: codice_hi_h_azimuth - VAR_NOTES: Energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. + LABL_PTR_4: codice_hi_h_azimuth + DISPLAY_TYPE: no_plot + FORMAT: F16.6 + VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. codice_lo_c_over_o_abundance: <<: *default_float32 diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 59ad4cfe80..b08f48d2e2 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -77,7 +77,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) swe_electron_energy_labels = xr.DataArray( - [ + data=[ "100.4 eV", "140 eV", "194 eV", @@ -95,7 +95,23 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) energy_range = xr.DataArray( - data=np.arange(15, dtype=np.uint8), + data=[ + "0.0200-0.0283 MeV", + "0.0283-0.0400 MeV", + "0.0400-0.0566 MeV", + "0.0566-0.0800 MeV", + "0.0800-0.113 MeV", + "0.113-0.160 MeV", + "0.160-0.226 MeV", + "0.226-0.320 MeV", + "0.320-0.453 MeV", + "0.453-0.640 MeV", + "0.640-0.905 MeV", + "0.905-1.28 MeV", + "1.28-1.81 MeV", + "1.81-2.56 MeV", + "2.56-3.62 MeV", + ], name="codice_hi_h_energy_range", dims=["codice_hi_h_energy_range"], attrs=cdf_manager.get_variable_attributes( @@ -104,7 +120,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) azimuth = xr.DataArray( - data=np.arange(4, dtype=np.uint8), + data=["TODO", "TODO", "TODO", "TODO"], name="codice_hi_h_azimuth", dims=["codice_hi_h_azimuth"], attrs=cdf_manager.get_variable_attributes( @@ -113,7 +129,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) spin_angle = xr.DataArray( - data=np.arange(4, dtype=np.uint8), + data=["TODO", "TODO", "TODO", "TODO"], name="codice_hi_h_spin_angle", dims=["codice_hi_h_spin_angle"], attrs=cdf_manager.get_variable_attributes( @@ -122,7 +138,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) spin_sector = xr.DataArray( - data=np.arange(4, dtype=np.uint8), + data=["TODO", "TODO", "TODO", "TODO"], name="codice_hi_h_spin_sector", dims=["codice_hi_h_spin_sector"], attrs=cdf_manager.get_variable_attributes( From bd67c57bf29df43cadb1e5cf995ad794eb367af1 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 16:58:07 -0700 Subject: [PATCH 12/42] add epoch --- .../config/imap_ialirt_l1_variable_attrs.yaml | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 9d8d36a97f..37cd840dfa 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -148,11 +148,48 @@ default_float64_attrs: &default_float64 VALIDMAX: 1.7976931348623157e+308 dtype: float64 +codice_hi_epoch: + CATDESC: CoDICE-Hi instrument epoch time tag + FIELDNAM: CoDICE-Hi epoch + LABLAXIS: CoDICE-Hi epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) + VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + RESOLUTION: ' ' + DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" + +codice_lo_epoch: + CATDESC: CoDICE-Lo instrument epoch time tag + FIELDNAM: CoDICE-Lo epoch + LABLAXIS: CoDICE-Lo epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) + VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + RESOLUTION: ' ' + DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" + codice_hi_h: <<: *default_float32 CATDESC: Proton intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins FIELDNAM: H intensity LABLAXIS: H intensity + DEPEND_0: codice_hi_epoch UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -173,6 +210,7 @@ codice_lo_c_over_o_abundance: CATDESC: C/O abundance ratio FIELDNAM: C/O abundance ratio LABLAXIS: C/O ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -183,6 +221,7 @@ codice_lo_mg_over_o_abundance: CATDESC: Mg/O abundance ratio FIELDNAM: Mg/O abundance ratio LABLAXIS: Mg/O ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -193,6 +232,7 @@ codice_lo_fe_over_o_abundance: CATDESC: Fe/O abundance ratio FIELDNAM: Fe/O abundance ratio LABLAXIS: Fe/O ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -203,6 +243,7 @@ codice_lo_c_plus_6_over_c_plus_5_ratio: CATDESC: C+6/C+5 charge state ratio FIELDNAM: C+6/C+5 charge state ratio LABLAXIS: C+6/C+5 ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -213,6 +254,7 @@ codice_lo_o_plus_7_over_o_plus_6_ratio: CATDESC: O+7/O+6 charge state ratio FIELDNAM: O7+/O6+ charge state ratio LABLAXIS: O7+/O6+ ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -223,6 +265,7 @@ codice_lo_fe_low_over_fe_high_ratio: CATDESC: Fe low/high charge state ratio (low=6–12, high=13–17) FIELDNAM: Fe low/high charge state ratio LABLAXIS: Fe low/high ratio + DEPEND_0: codice_lo_epoch UNITS: " " VALIDMIN: 0 VALIDMAX: 100000000.0 @@ -349,19 +392,29 @@ hit_he_omni_high_en: VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. mag_epoch: - <<: *default_float32 CATDESC: MAG instrument epoch time tag FIELDNAM: MAG epoch LABLAXIS: MAG epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) + VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end UNITS: ns - VALIDMIN: 0.0 - VALIDMAX: 9223372036854775807.0 + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + RESOLUTION: ' ' + DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" mag_B_GSE: <<: *default_float32 CATDESC: Magnetic field vector in GSE coordinates FIELDNAM: B vector in GSE coordinates LABLAXIS: B (GSE) + DEPEND_0: mag_epoch LABL_PTR_1: B_GSE_labels UNITS: nT FORMAT: F13.6 @@ -373,6 +426,7 @@ mag_B_GSM: CATDESC: Magnetic field vector in GSM coordinates FIELDNAM: B vector in GSM coordinates LABLAXIS: B (GSM) + DEPEND_0: mag_epoch LABL_PTR_1: B_GSM_labels UNITS: nT FORMAT: F13.6 @@ -384,6 +438,7 @@ mag_B_RTN: CATDESC: Magnetic field vector in RTN coordinates FIELDNAM: B vector in RTN coordinates LABLAXIS: B (RTN) + DEPEND_0: mag_epoch LABL_PTR_1: B_RTN_labels UNITS: nT FORMAT: F13.6 @@ -395,6 +450,7 @@ mag_B_magnitude: CATDESC: Magnitude of the magnetic field vector FIELDNAM: B vector magnitude LABLAXIS: B Magnitude + DEPEND_0: mag_epoch UNITS: nT VALIDMIN: 0.0 VALIDMAX: 65000.0 @@ -404,6 +460,7 @@ mag_phi_B_GSM: CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates FIELDNAM: B azimuth angle in GSM coordinates LABLAXIS: B azimuth (GSM) + DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: 0.0 VALIDMAX: 360.0 @@ -413,6 +470,7 @@ mag_theta_B_GSM: CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates FIELDNAM: B elevation angle in GSM coordinates LABLAXIS: B elevation (GSM) + DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: -90.0 VALIDMAX: 90.0 @@ -422,6 +480,7 @@ mag_phi_B_GSE: CATDESC: Azimuth angle (φ) of the magnetic field in GSE coordinates FIELDNAM: B azimuth angle in GSE coordinates LABLAXIS: B azimuth (GSE) + DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: 0.0 VALIDMAX: 360.0 @@ -431,6 +490,7 @@ mag_theta_B_GSE: CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates FIELDNAM: B elevation angle in GSE coordinates LABLAXIS: B elevation (GSE) + DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: -90.0 VALIDMAX: 90.0 From 059c67153292880f390d6d9f590fbf1652090a5a Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 17:23:47 -0700 Subject: [PATCH 13/42] improve time --- imap_processing/ialirt/l0/parse_mag.py | 4 +++- imap_processing/ialirt/l0/process_codice.py | 9 +++++--- imap_processing/ialirt/l0/process_hit.py | 10 +++++---- imap_processing/ialirt/l0/process_swapi.py | 12 +++++------ imap_processing/ialirt/l0/process_swe.py | 24 ++++++++++++--------- 5 files changed, 35 insertions(+), 24 deletions(-) diff --git a/imap_processing/ialirt/l0/parse_mag.py b/imap_processing/ialirt/l0/parse_mag.py index a0500c07dd..f715d0b972 100644 --- a/imap_processing/ialirt/l0/parse_mag.py +++ b/imap_processing/ialirt/l0/parse_mag.py @@ -716,7 +716,9 @@ def process_packet( ) met = grouped_data["met"][(grouped_data["group"] == group).values] - met_all.append(met.values[0]) + # Calculate the center of the time variable + sc_met = int(np.mean([np.min(met), np.max(met)])) + met_all.append(sc_met) mago_times_all.append(time_data["primary_epoch"]) mago_vectors_all.append(mago_inertial_vector) magi_vectors_all.append(magi_inertial_vector) diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 401d428390..1dcef0961b 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -513,12 +513,15 @@ def process_codice( l2_hi ).tolist() + # Calculate the center of the time variable + sc_met = int(np.mean([np.min(met), np.max(met)])) + codice_hi_data.append( { "apid": 478, - "met": int(met[0]), - "met_in_utc": met_to_utc(met[0]).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met[0])), + "met": int(sc_met), + "met_in_utc": met_to_utc(sc_met).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", f"{sensor}_epoch": [int(epoch) for epoch in l1b_hi["epoch"]], f"{sensor}_h": dec_l2_hi, diff --git a/imap_processing/ialirt/l0/process_hit.py b/imap_processing/ialirt/l0/process_hit.py index 8aaf42a87c..baa68253fb 100644 --- a/imap_processing/ialirt/l0/process_hit.py +++ b/imap_processing/ialirt/l0/process_hit.py @@ -161,16 +161,18 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: slow_rate = grouped_data["hit_slow_rate"][ (grouped_data["group"] == group).values ] - met = int(grouped_data["met"][(grouped_data["group"] == group).values][0]) + met = grouped_data["met"][(grouped_data["group"] == group).values] + # Calculate the center of the time variable + sc_met = int(np.mean([np.min(met), np.max(met)])) l1 = create_l1(fast_rate_1, fast_rate_2, slow_rate) hit_data.append( { "apid": 478, - "met": int(met), - "met_in_utc": met_to_utc(met).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met)), + "met": sc_met, + "met_in_utc": met_to_utc(sc_met).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": "hit", "hit_e_a_side_low_en": int(l1["IALRT_RATE_1"] + l1["IALRT_RATE_2"]), "hit_e_a_side_med_en": int(l1["IALRT_RATE_5"] + l1["IALRT_RATE_6"]), diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 239eccdbbe..175063ef49 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -162,9 +162,9 @@ def process_swapi_ialirt( (grouped_dataset["group"] == group) ] - met_values = int( - grouped_dataset["met"][(grouped_dataset["group"] == group).values][0] - ) + met = grouped_dataset["met"][(grouped_dataset["group"] == group).values] + # Calculate the center of the time variable + sc_met = int(np.mean([np.min(met), np.max(met)])) # Ensure no duplicates and all values from 0 to 11 are present if not np.array_equal(seq_values.values.astype(int), np.arange(12)): @@ -208,9 +208,9 @@ def process_swapi_ialirt( swapi_data.append( { "apid": 478, - "met": int(met_values), - "met_in_utc": met_to_utc(met_values).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met_values)), + "met": sc_met, + "met_in_utc": met_to_utc(sc_met).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": "swapi", "swapi_pseudo_proton_speed": Decimal(f"{pseudo_speed:.3f}"), "swapi_pseudo_proton_density": Decimal(f"{pseudo_density:.3f}"), diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 763b731e44..f78513b711 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -548,19 +548,23 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list summed_first = normalized_first_half.sum(axis=(1, 2)) summed_second = normalized_second_half.sum(axis=(1, 2)) - met_first_half = int( - grouped["met"].where(grouped["swe_seq"] == 0, drop=True).values[0] + met_first_half = grouped["met"].where(grouped["swe_seq"] == 0, drop=True).values + sc_met_first_half = int( + np.mean([np.min(met_first_half), np.max(met_first_half)]) ) - met_second_half = int( - grouped["met"].where(grouped["swe_seq"] == 30, drop=True).values[0] + met_second_half = ( + grouped["met"].where(grouped["swe_seq"] == 30, drop=True).values + ) + sc_met_second_half = int( + np.mean([np.min(met_second_half), np.max(met_second_half)]) ) swe_data.append( { "apid": 478, - "met": met_first_half, - "met_in_utc": met_to_utc(met_first_half).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met_first_half)), + "met": sc_met_first_half, + "met_in_utc": met_to_utc(sc_met_first_half).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met_first_half)), "instrument": "swe", "swe_normalized_counts": [int(val) for val in summed_first], "swe_counterstreaming_electrons": bde_first_half, @@ -569,9 +573,9 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list swe_data.append( { "apid": 478, - "met": met_second_half, - "met_in_utc": met_to_utc(met_second_half).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met_second_half)), + "met": sc_met_second_half, + "met_in_utc": met_to_utc(sc_met_second_half).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met_second_half)), "instrument": "swe", "swe_normalized_counts": [int(val) for val in summed_second], "swe_counterstreaming_electrons": bde_second_half, From 068a7f020cbc8ff62c3fa8fdbe1b692d4cbacd6b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 17:33:31 -0700 Subject: [PATCH 14/42] update time --- .../config/imap_ialirt_l1_variable_attrs.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 37cd840dfa..b88f9b5102 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -166,24 +166,6 @@ codice_hi_epoch: RESOLUTION: ' ' DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" -codice_lo_epoch: - CATDESC: CoDICE-Lo instrument epoch time tag - FIELDNAM: CoDICE-Lo epoch - LABLAXIS: CoDICE-Lo epoch - FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf - VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) - VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid - RESOLUTION: ' ' - DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" - codice_hi_h: <<: *default_float32 CATDESC: Proton intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins From 743c35c1b44e970d98b1779e65f161f6c73338af Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 6 Jan 2026 17:37:07 -0700 Subject: [PATCH 15/42] update pseudo description --- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index b88f9b5102..abadfe35d3 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -525,7 +525,7 @@ swapi_pseudo_proton_density: UNITS: 1/cm^3 VALIDMIN: 0.0 VALIDMAX: 1000.0 - VAR_NOTES: The pseudo density is derived from the SWAPI proton moments but does not account for instrument calibration factors. + VAR_NOTES: The pseudo density is derived using a simplified analytical model of solar wind proton density. swapi_pseudo_proton_speed: <<: *default_float32 @@ -535,7 +535,7 @@ swapi_pseudo_proton_speed: UNITS: km/sec VALIDMIN: 0.0 VALIDMAX: 10000.0 - VAR_NOTES: The pseudo speed is derived from the SWAPI proton moments but does not account for instrument calibration factors. + VAR_NOTES: The pseudo speed is derived using a simplified analytical model of solar wind proton speed. swapi_pseudo_proton_temperature: <<: *default_float32 @@ -545,7 +545,7 @@ swapi_pseudo_proton_temperature: UNITS: Kelvin VALIDMIN: 0.0 VALIDMAX: 50000000.0 - VAR_NOTES: The pseudo temperature is derived from the SWAPI proton moments but does not account for instrument calibration factors. + VAR_NOTES: The pseudo temperature is derived using a simplified analytical model of solar wind proton speed. # SWE Normalized Counts swe_normalized_counts: From c5afe5e5d52db2f1a0ffc8296da558841d62608f Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 7 Jan 2026 14:33:32 -0700 Subject: [PATCH 16/42] stopping point --- .../config/imap_ialirt_l1_variable_attrs.yaml | 78 +++++++++++-------- imap_processing/ialirt/l0/process_codice.py | 3 + imap_processing/ialirt/utils/create_xarray.py | 62 +++++++++++---- .../tests/ialirt/unit/test_create_xarray.py | 17 +++- 4 files changed, 109 insertions(+), 51 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index abadfe35d3..ec399a9083 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -61,37 +61,6 @@ swe_electron_energy_labels: VAR_TYPE: support_data UNITS: " " -codice_hi_h_spin_angle: - CATDESC: CoDICE-Hi spin angle bin - FIELDNAM: CoDICE-Hi spin angle bin - LABLAXIS: Spin angle bin - FORMAT: A20 - VAR_TYPE: support_data - UNITS: "TODO" - -codice_hi_h_energy_range: - CATDESC: CoDICE-Hi proton energy bin labels (MeV) - FIELDNAM: CoDICE-Hi proton energy bin labels - FORMAT: A20 - VAR_TYPE: support_data - UNITS: "MeV" - -codice_hi_h_spin_sector: - CATDESC: CoDICE-Hi spin sector - FIELDNAM: CoDICE-Hi spin sector - LABLAXIS: Spin sector - FORMAT: A20 - VAR_TYPE: support_data - UNITS: "TODO" - -codice_hi_h_azimuth: - CATDESC: CoDICE-Hi azimuth bin - FIELDNAM: CoDICE-Hi azimuth bin - LABLAXIS: Azimuth bin - FORMAT: A20 - VAR_TYPE: support_data - UNITS: "TODO" - default_uint8_attrs: &default_uint8 <<: *default FILLVAL: 255 @@ -148,6 +117,47 @@ default_float64_attrs: &default_float64 VALIDMAX: 1.7976931348623157e+308 dtype: float64 +codice_hi_h_spin_angle: + <<: *default_float32 + CATDESC: CoDICE-Hi spin angle + FIELDNAM: CoDICE-Hi spin angle + LABLAXIS: Spin angle + SCALETYP: linear + UNITS: degrees + VALIDMIN: 0.0 + VALIDMAX: 180.0 + VAR_TYPE: support_data + +codice_hi_h_energy_range: + CATDESC: CoDICE-Hi proton energy bin labels (MeV) + FIELDNAM: CoDICE-Hi proton energy bin labels + FORMAT: A20 + VAR_TYPE: support_data + UNITS: "MeV" + +codice_hi_h_spin_sector: + CATDESC: CoDICE-Hi spin sector index + FIELDNAM: CoDICE-Hi spin sector index + FILLVAL: -1 + FORMAT: I2 + LABLAXIS: " " + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 3 + VAR_TYPE: support_data + +codice_hi_h_elevation: + <<: *default_float32 + CATDESC: Elevation Angle + FIELDNAM: Elevation Angle + LABLAXIS: Elevation Angle + SCALETYP: linear + UNITS: degrees + VALIDMIN: 0.0 + VALIDMAX: 180.0 + VAR_TYPE: support_data + codice_hi_epoch: CATDESC: CoDICE-Hi instrument epoch time tag FIELDNAM: CoDICE-Hi epoch @@ -168,7 +178,7 @@ codice_hi_epoch: codice_hi_h: <<: *default_float32 - CATDESC: Proton intensity in 4 spin angle bins, 15 energy ranges, 4 spin sector bins, and 4 azimuth bins + CATDESC: Proton intensity in 4 spin angles, 15 energy ranges, 4 spin sectors, and 4 elevations FIELDNAM: H intensity LABLAXIS: H intensity DEPEND_0: codice_hi_epoch @@ -181,8 +191,8 @@ codice_hi_h: LABL_PTR_2: codice_hi_h_energy_range DEPEND_3: codice_hi_h_spin_sector LABL_PTR_3: codice_hi_h_spin_sector - DEPEND_4: codice_hi_h_azimuth - LABL_PTR_4: codice_hi_h_azimuth + DEPEND_4: codice_hi_h_elevation + LABL_PTR_4: codice_hi_h_elevation DISPLAY_TYPE: no_plot FORMAT: F16.6 VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 1dcef0961b..86f468fbfd 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -524,6 +524,9 @@ def process_codice( "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", f"{sensor}_epoch": [int(epoch) for epoch in l1b_hi["epoch"]], + # TODO: add values here. How? + # codice_hi_h_spin_angle (n=4) + # codice_hi_h_elevation_angle (n=4) f"{sensor}_h": dec_l2_hi, } ) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index b08f48d2e2..c471c1ae85 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -29,10 +29,22 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs = cdf_manager.get_variable_attributes("default_int64_attrs") fillval = attrs.get("FILLVAL") ttj2000ns_values = np.full(n, fillval, dtype=np.int64) + codice_hi_ttj2000ns_values = np.full(n * 4, fillval, dtype=np.int64) + mag_ttj2000ns_values = np.full(n * 4, fillval, dtype=np.int64) + attrs = cdf_manager.get_variable_attributes("default_float32_attrs") + fillval = attrs.get("FILLVAL") + elevation_values = np.full(n * 4, fillval, dtype=np.float32) + spin_angle_values = np.full(n * 4, fillval, dtype=np.float32) # Collect all keys that start with the instrument prefixes. for i, record in enumerate(records): ttj2000ns_values[i] = record["ttj2000ns"] + if record["instrument"] == "codice_hi": + codice_hi_ttj2000ns_values[4 * i : 4 * i + 4] = record["codice_hi_epoch"] + elevation_values[4 * i : 4 * i + 4] = record["codice_hi_h_elevation_angle"] + spin_angle_values[4 * i : 4 * i + 4] = record["codice_hi_h_spin_angle"] + if record["instrument"] == "mag": + mag_ttj2000ns_values[4 * i : 4 * i + 4] = record["mag_epoch"] epoch = xr.DataArray( data=ttj2000ns_values, @@ -41,6 +53,22 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False), ) + codice_hi_epoch = xr.DataArray( + data=codice_hi_ttj2000ns_values, + name="codice_hi_epoch", + dims=["codice_hi_epoch"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_epoch", check_schema=False + ), + ) + + mag_epoch = xr.DataArray( + data=mag_ttj2000ns_values, + name="mag_epoch", + dims=["mag_epoch"], + attrs=cdf_manager.get_variable_attributes("mag_epoch", check_schema=False), + ) + sc_gsm_component = xr.DataArray( ["x (GSM)", "y (GSM)", "z (GSM)"], name="sc_GSM_labels", @@ -119,17 +147,19 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - azimuth = xr.DataArray( - data=["TODO", "TODO", "TODO", "TODO"], - name="codice_hi_h_azimuth", - dims=["codice_hi_h_azimuth"], + # TODO: I think this need to be indices too. + elevation = xr.DataArray( + data=elevation_values, + name="codice_hi_h_elevation", + dims=["codice_hi_h_elevation"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_azimuth", check_schema=False + "codice_hi_h_elevation", check_schema=False ), ) + # TODO: I think this need to be indices too. spin_angle = xr.DataArray( - data=["TODO", "TODO", "TODO", "TODO"], + data=spin_angle_values, name="codice_hi_h_spin_angle", dims=["codice_hi_h_spin_angle"], attrs=cdf_manager.get_variable_attributes( @@ -138,7 +168,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) spin_sector = xr.DataArray( - data=["TODO", "TODO", "TODO", "TODO"], + data=np.arange(4, dtype=np.uint8), name="codice_hi_h_spin_sector", dims=["codice_hi_h_spin_sector"], attrs=cdf_manager.get_variable_attributes( @@ -148,6 +178,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 coords = { "epoch": epoch, + "mag_epoch": mag_epoch, + "codice_hi_epoch": codice_hi_epoch, "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, @@ -157,7 +189,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "codice_hi_h_spin_angle": spin_angle, "codice_hi_h_energy_range": energy_range, "codice_hi_h_spin_sector": spin_sector, - "codice_hi_h_azimuth": azimuth, + "codice_hi_h_elevation": elevation, } dataset = xr.Dataset( coords=coords, @@ -170,7 +202,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 fillval = attrs.get("FILLVAL") if key == "mag_B_GSE": data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "B_GSE_labels"] + dims = ["mag_epoch", "B_GSE_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key in ["sc_position_GSE", "sc_velocity_GSE"]: data = np.full((n, 3), fillval, dtype=np.float32) @@ -182,20 +214,20 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "mag_B_GSM": data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "B_GSM_labels"] + dims = ["mag_epoch", "B_GSM_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "mag_B_RTN": data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "B_RTN_labels"] + dims = ["mag_epoch", "B_RTN_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("codice_hi"): data = np.full((n, 4, 15, 4, 4), fillval, dtype=np.float32) dims = [ - "epoch", - "codice_hi_h_spin_angle", + "codice_hi_epoch", # ? not certain this is correct. + "codice_hi_h_spin_angle", # changes within each group "codice_hi_h_energy_range", - "codice_hi_h_spin_sector", - "codice_hi_h_azimuth", + "codice_hi_h_spin_sector", # index (0, 1, 2, 3) + "codice_hi_h_elevation", # static for each group (btw groups changes) ] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "swe_counterstreaming_electrons": diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 123ee2b188..1f788206ca 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -17,6 +17,7 @@ def test_create_dataset(): "met": 123456789, "met_in_utc": "2025-06-20T08:00:00", "ttj2000ns": 123456789000000, + "instrument": "codice_hi", "swe_normalized_counts": [Decimal("0.0") for _ in range(8)], "swe_counterstreaming_electrons": Decimal("0.0"), "swapi_pseudo_proton_speed": Decimal("0.0"), @@ -52,13 +53,25 @@ def test_create_dataset(): [[Decimal("0.0") for _ in range(4)] for _ in range(4)] for _ in range(15) ], + "codice_hi_h_spin_angle": [ + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + ], + "codice_hi_h_elevation_angle": [ + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + ], }, { "apid": 478, "met": 123456789, "met_in_utc": "2025-06-20T08:00:00", "ttj2000ns": 123456789000001, - # Only MAG is present + "instrument": "mag", "mag_epoch": Decimal("0.0"), "mag_B_GSE": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], "mag_B_GSM": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], @@ -74,7 +87,7 @@ def test_create_dataset(): "met": 123456789, "met_in_utc": "2025-06-20T08:00:00", "ttj2000ns": 123456789000002, - # Only SWAPI is present + "instrument": "swapi", "swapi_pseudo_proton_speed": Decimal("0.0"), "swapi_pseudo_proton_density": Decimal("0.0"), "swapi_pseudo_proton_temperature": Decimal("0.0"), From 9a63c96ad8f182f6efb7fa6f0030950dd70bddc1 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 11:02:51 -0700 Subject: [PATCH 17/42] codice update --- .../config/imap_ialirt_l1_variable_attrs.yaml | 57 +++++---- imap_processing/codice/constants.py | 18 +++ imap_processing/ialirt/l0/process_codice.py | 3 - imap_processing/ialirt/utils/constants.py | 29 +++++ imap_processing/ialirt/utils/create_xarray.py | 116 ++++++++++-------- .../tests/ialirt/unit/test_create_xarray.py | 6 + 6 files changed, 153 insertions(+), 76 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index ec399a9083..8802ca8dad 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -117,25 +117,14 @@ default_float64_attrs: &default_float64 VALIDMAX: 1.7976931348623157e+308 dtype: float64 -codice_hi_h_spin_angle: - <<: *default_float32 - CATDESC: CoDICE-Hi spin angle - FIELDNAM: CoDICE-Hi spin angle - LABLAXIS: Spin angle - SCALETYP: linear - UNITS: degrees - VALIDMIN: 0.0 - VALIDMAX: 180.0 - VAR_TYPE: support_data - -codice_hi_h_energy_range: +codice_hi_h_energy_range_labels: CATDESC: CoDICE-Hi proton energy bin labels (MeV) FIELDNAM: CoDICE-Hi proton energy bin labels FORMAT: A20 VAR_TYPE: support_data UNITS: "MeV" -codice_hi_h_spin_sector: +codice_hi_spin_sector: CATDESC: CoDICE-Hi spin sector index FIELDNAM: CoDICE-Hi spin sector index FILLVAL: -1 @@ -147,7 +136,7 @@ codice_hi_h_spin_sector: VALIDMAX: 3 VAR_TYPE: support_data -codice_hi_h_elevation: +codice_hi_elevation: <<: *default_float32 CATDESC: Elevation Angle FIELDNAM: Elevation Angle @@ -158,6 +147,34 @@ codice_hi_h_elevation: VALIDMAX: 180.0 VAR_TYPE: support_data +codice_hi_elevation_labels: + CATDESC: Elevation Angle + FIELDNAM: Elevation Angle + FORMAT: A6 + VAR_TYPE: metadata + +codice_hi_spin_sector_labels: + CATDESC: Spin Sector + DEPEND_1: spin_sector + FIELDNAM: Spin Sector + FORMAT: A2 + VAR_TYPE: metadata + +codice_hi_spin_angle: + <<: *default_float32 + VAR_TYPE: support_data + CATDESC: Spin Angle + FIELDNAM: Spin Angle + SCALETYP: linear + UNITS: degrees + VALIDMAX: 360.0 + VALIDMIN: 0.0 + DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle + DEPEND_1: codice_hi_spin_sector + DEPEND_2: codice_hi_elevation + LABL_PTR_1: codice_hi_spin_sector_labels + LABL_PTR_2: codice_hi_elevation_labels + codice_hi_epoch: CATDESC: CoDICE-Hi instrument epoch time tag FIELDNAM: CoDICE-Hi epoch @@ -185,14 +202,10 @@ codice_hi_h: UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 - DEPEND_1: codice_hi_h_spin_angle - LABL_PTR_1: codice_hi_h_spin_angle - DEPEND_2: codice_hi_h_energy_range - LABL_PTR_2: codice_hi_h_energy_range - DEPEND_3: codice_hi_h_spin_sector - LABL_PTR_3: codice_hi_h_spin_sector - DEPEND_4: codice_hi_h_elevation - LABL_PTR_4: codice_hi_h_elevation + LABL_PTR_1: codice_hi_h_energy_range_labels + DEPEND_2: codice_hi_h_spin_sector_labels + DEPEND_3: codice_hi_h_elevation + LABL_PTR_3: codice_hi_h_elevation_labels DISPLAY_TYPE: no_plot FORMAT: F16.6 VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index f108779196..467e46198e 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -76,6 +76,24 @@ "fe_hiq": 7.25, } +HI_IALIRT_ELEVATION_ANGLE = np.array( + [ + 132.8, + 65.7, + 47.1, + 114.3, + ], + dtype=float, +) +HI_IALIRT_REF_SPIN_ANGLE = np.array( + [ + 286.85, + 264.55, + 343.16, + 5.44, + ], + dtype=float, +) # Define the packet fields needed to be stored in segmented data and their # corresponding bit lengths for direct event data products diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 86f468fbfd..1dcef0961b 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -524,9 +524,6 @@ def process_codice( "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", f"{sensor}_epoch": [int(epoch) for epoch in l1b_hi["epoch"]], - # TODO: add values here. How? - # codice_hi_h_spin_angle (n=4) - # codice_hi_h_elevation_angle (n=4) f"{sensor}_h": dec_l2_hi, } ) diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 2f73f0055c..7c50686dd4 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -68,3 +68,32 @@ # Spacecraft velocity in GSE coordinates "sc_velocity_GSE", ] + +codice_energy_bounds = [ + (0.020, 0.028), + (0.028, 0.040), + (0.040, 0.057), + (0.057, 0.080), + (0.080, 0.113), + (0.113, 0.160), + (0.160, 0.226), + (0.226, 0.320), + (0.320, 0.453), + (0.453, 0.640), + (0.640, 0.905), + (0.905, 1.280), + (1.280, 1.810), + (1.810, 2.560), + (2.560, 3.620), +] + +swe_energy_labels = [ + "100.4", + "140.0", + "194.0", + "270.0", + "376.0", + "523.0", + "727.0", + "1011.0", +] diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index c471c1ae85..d67d4dddb4 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -4,7 +4,15 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.ialirt.utils.constants import IALIRT_KEYS +from imap_processing.codice.constants import ( + HI_IALIRT_ELEVATION_ANGLE, + HI_IALIRT_REF_SPIN_ANGLE, +) +from imap_processing.ialirt.utils.constants import ( + IALIRT_KEYS, + codice_energy_bounds, + swe_energy_labels, +) def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0912 @@ -105,16 +113,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) swe_electron_energy_labels = xr.DataArray( - data=[ - "100.4 eV", - "140 eV", - "194 eV", - "270 eV", - "376 eV", - "523 eV", - "727 eV", - "1011 eV", - ], + data=[f"{label} eV" for label in swe_energy_labels], name="swe_electron_energy_labels", dims=["swe_electron_energy_labels"], attrs=cdf_manager.get_variable_attributes( @@ -123,47 +122,52 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) energy_range = xr.DataArray( - data=[ - "0.0200-0.0283 MeV", - "0.0283-0.0400 MeV", - "0.0400-0.0566 MeV", - "0.0566-0.0800 MeV", - "0.0800-0.113 MeV", - "0.113-0.160 MeV", - "0.160-0.226 MeV", - "0.226-0.320 MeV", - "0.320-0.453 MeV", - "0.453-0.640 MeV", - "0.640-0.905 MeV", - "0.905-1.28 MeV", - "1.28-1.81 MeV", - "1.81-2.56 MeV", - "2.56-3.62 MeV", - ], - name="codice_hi_h_energy_range", - dims=["codice_hi_h_energy_range"], + data=[f"{low:.4f}-{high:.4f} MeV" for low, high in codice_energy_bounds], + name="codice_hi_h_energy_range_labels", + dims=["codice_hi_h_energy_range_labels"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_energy_range", check_schema=False + "codice_hi_h_energy_range_labels", check_schema=False ), ) - # TODO: I think this need to be indices too. - elevation = xr.DataArray( - data=elevation_values, - name="codice_hi_h_elevation", - dims=["codice_hi_h_elevation"], + elevation = ( + xr.DataArray( + HI_IALIRT_ELEVATION_ANGLE, + name="codice_hi_elevation", + dims=("codice_hi_elevation",), + attrs=cdf_manager.get_variable_attributes( + "codice_hi_elevation", check_schema=False + ), + ), + ) + + elevation_labels = xr.DataArray( + HI_IALIRT_ELEVATION_ANGLE.astype(str), + name="codice_hi_elevation_labels", + dims=["codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_elevation", check_schema=False + "codice_hi_elevation_labels", check_schema=False ), ) - # TODO: I think this need to be indices too. + # Calculate spin angle for CoDICE-Hi + # Formula: + # θ_(g,n) = (θ_(g,0)+90°* n) mod 360° + # where + # n is number of sectored angles, 0 to 3, + # g is size of the group (inst_az), 0 to 3, + # Calculate spin angle by adding a base angle from L2_HI_SECTORED_ANGLE + # for each SSD index and then adding multiple of 30 degrees for each elevation. + # Then mod by 360 to keep it within 0-360 range. + spin_angles = ( + HI_IALIRT_REF_SPIN_ANGLE[:, np.newaxis] + np.array([0, 1, 2, 3]) * 90 + ) % 360.0 spin_angle = xr.DataArray( - data=spin_angle_values, - name="codice_hi_h_spin_angle", - dims=["codice_hi_h_spin_angle"], + data=spin_angles, + name="codice_hi_spin_angle", + dims=["spin_sector", "elevation_angle"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_spin_angle", check_schema=False + "codice_hi_spin_angle", check_schema=False ), ) @@ -176,6 +180,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) + spin_sector_labels = xr.DataArray( + data=["0", "1", "2", "3"], + name="codice_hi_h_spin_sector_labels", + dims=["codice_hi_h_spin_sector"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_h_spin_sector_labels", check_schema=False + ), + ) + coords = { "epoch": epoch, "mag_epoch": mag_epoch, @@ -186,10 +199,12 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "sc_GSM_labels": sc_gsm_component, "sc_GSE_labels": sc_gse_component, "swe_electron_energy_labels": swe_electron_energy_labels, - "codice_hi_h_spin_angle": spin_angle, "codice_hi_h_energy_range": energy_range, - "codice_hi_h_spin_sector": spin_sector, - "codice_hi_h_elevation": elevation, + "codice_hi_elevation": elevation, + "codice_hi_elevation_labels": elevation_labels, + "codice_hi_spin_angle": spin_angle, + "codice_hi_spin_sector": spin_sector, + "codice_hi_spin_sector_labels": spin_sector_labels, } dataset = xr.Dataset( coords=coords, @@ -221,13 +236,12 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dims = ["mag_epoch", "B_RTN_labels"] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key.startswith("codice_hi"): - data = np.full((n, 4, 15, 4, 4), fillval, dtype=np.float32) + data = np.full((4 * n, 15, 4, 4), fillval, dtype=np.float32) dims = [ - "codice_hi_epoch", # ? not certain this is correct. - "codice_hi_h_spin_angle", # changes within each group - "codice_hi_h_energy_range", - "codice_hi_h_spin_sector", # index (0, 1, 2, 3) - "codice_hi_h_elevation", # static for each group (btw groups changes) + "codice_hi_epoch", + "codice_hi_energy_range", + "codice_hi_spin_sector", + "codice_hi_elevation", ] dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) elif key == "swe_counterstreaming_electrons": diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 1f788206ca..9a6de42371 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -53,6 +53,12 @@ def test_create_dataset(): [[Decimal("0.0") for _ in range(4)] for _ in range(4)] for _ in range(15) ], + "codice_hi_epoch": [ + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + ], "codice_hi_h_spin_angle": [ Decimal("0.0"), Decimal("0.0"), From 023ce9e7069f971fd4012690b80ff9181c5404f6 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 12:47:09 -0700 Subject: [PATCH 18/42] update --- imap_processing/ialirt/utils/create_xarray.py | 193 +++++------------- .../tests/ialirt/unit/test_create_xarray.py | 52 +---- 2 files changed, 64 insertions(+), 181 deletions(-) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index d67d4dddb4..0ae86bfaa4 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -33,49 +33,28 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 cdf_manager.add_instrument_global_attrs("ialirt") cdf_manager.add_instrument_variable_attrs("ialirt", "l1") - n = len(records) - attrs = cdf_manager.get_variable_attributes("default_int64_attrs") - fillval = attrs.get("FILLVAL") - ttj2000ns_values = np.full(n, fillval, dtype=np.int64) - codice_hi_ttj2000ns_values = np.full(n * 4, fillval, dtype=np.int64) - mag_ttj2000ns_values = np.full(n * 4, fillval, dtype=np.int64) - attrs = cdf_manager.get_variable_attributes("default_float32_attrs") - fillval = attrs.get("FILLVAL") - elevation_values = np.full(n * 4, fillval, dtype=np.float32) - spin_angle_values = np.full(n * 4, fillval, dtype=np.float32) + ONE_EPOCH = {"codice_lo", "hit", "swapi", "swe", "spacecraft", "mag"} + MULTI_EPOCH = {"codice_hi"} - # Collect all keys that start with the instrument prefixes. - for i, record in enumerate(records): - ttj2000ns_values[i] = record["ttj2000ns"] - if record["instrument"] == "codice_hi": - codice_hi_ttj2000ns_values[4 * i : 4 * i + 4] = record["codice_hi_epoch"] - elevation_values[4 * i : 4 * i + 4] = record["codice_hi_h_elevation_angle"] - spin_angle_values[4 * i : 4 * i + 4] = record["codice_hi_h_spin_angle"] - if record["instrument"] == "mag": - mag_ttj2000ns_values[4 * i : 4 * i + 4] = record["mag_epoch"] + epochs = {inst: [] for inst in (ONE_EPOCH | MULTI_EPOCH)} - epoch = xr.DataArray( - data=ttj2000ns_values, - name="epoch", - dims=["epoch"], - attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False), - ) + for r in records: + inst = r.get("instrument") + if inst in ONE_EPOCH: + epochs[inst].append(r["mag_epoch"] if inst == "mag" else r["ttj2000ns"]) + elif inst in MULTI_EPOCH: + epochs[inst].extend(r["codice_hi_epoch"]) - codice_hi_epoch = xr.DataArray( - data=codice_hi_ttj2000ns_values, - name="codice_hi_epoch", - dims=["codice_hi_epoch"], - attrs=cdf_manager.get_variable_attributes( - "codice_hi_epoch", check_schema=False - ), - ) + epoch_arrays = {} - mag_epoch = xr.DataArray( - data=mag_ttj2000ns_values, - name="mag_epoch", - dims=["mag_epoch"], - attrs=cdf_manager.get_variable_attributes("mag_epoch", check_schema=False), - ) + for inst, arr in epochs.items(): + coord = f"{inst}_epoch" + epoch_arrays[coord] = xr.DataArray( + data=arr, + name=coord, + dims=[coord], + attrs=cdf_manager.get_variable_attributes(coord, check_schema=False), + ) sc_gsm_component = xr.DataArray( ["x (GSM)", "y (GSM)", "z (GSM)"], @@ -130,24 +109,11 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - elevation = ( - xr.DataArray( - HI_IALIRT_ELEVATION_ANGLE, - name="codice_hi_elevation", - dims=("codice_hi_elevation",), - attrs=cdf_manager.get_variable_attributes( - "codice_hi_elevation", check_schema=False - ), - ), - ) - - elevation_labels = xr.DataArray( - HI_IALIRT_ELEVATION_ANGLE.astype(str), - name="codice_hi_elevation_labels", + elevation = xr.DataArray( + HI_IALIRT_ELEVATION_ANGLE, + name="codice_hi_elevation", dims=["codice_hi_elevation"], - attrs=cdf_manager.get_variable_attributes( - "codice_hi_elevation_labels", check_schema=False - ), + attrs=cdf_manager.get_variable_attributes("codice_hi_elevation", check_schema=False), ) # Calculate spin angle for CoDICE-Hi @@ -165,7 +131,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 spin_angle = xr.DataArray( data=spin_angles, name="codice_hi_spin_angle", - dims=["spin_sector", "elevation_angle"], + dims=["codice_hi_spin_sector", "codice_hi_elevation_angle"], attrs=cdf_manager.get_variable_attributes( "codice_hi_spin_angle", check_schema=False ), @@ -180,19 +146,13 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - spin_sector_labels = xr.DataArray( - data=["0", "1", "2", "3"], - name="codice_hi_h_spin_sector_labels", - dims=["codice_hi_h_spin_sector"], - attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_spin_sector_labels", check_schema=False - ), - ) - coords = { - "epoch": epoch, - "mag_epoch": mag_epoch, - "codice_hi_epoch": codice_hi_epoch, + "codice_hi_epoch": epoch_arrays["codice_hi_epoch"], + "codice_lo_epoch": epoch_arrays["codice_lo_epoch"], + "hit_epoch": epoch_arrays["hit_epoch"], + "mag_epoch": epoch_arrays["mag_epoch"], + "swapi_epoch": epoch_arrays["swapi_epoch"], + "swe_epoch": epoch_arrays["swe_epoch"], "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, @@ -201,87 +161,44 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "swe_electron_energy_labels": swe_electron_energy_labels, "codice_hi_h_energy_range": energy_range, "codice_hi_elevation": elevation, - "codice_hi_elevation_labels": elevation_labels, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, - "codice_hi_spin_sector_labels": spin_sector_labels, } dataset = xr.Dataset( coords=coords, attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"), ) - # Create empty dataset for each key. - for key in IALIRT_KEYS: - attrs = cdf_manager.get_variable_attributes(key, check_schema=False) - fillval = attrs.get("FILLVAL") - if key == "mag_B_GSE": - data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["mag_epoch", "B_GSE_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key in ["sc_position_GSE", "sc_velocity_GSE"]: - data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "sc_GSE_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key in ["sc_position_GSM", "sc_velocity_GSM"]: - data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["epoch", "sc_GSM_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key == "mag_B_GSM": - data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["mag_epoch", "B_GSM_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key == "mag_B_RTN": - data = np.full((n, 3), fillval, dtype=np.float32) - dims = ["mag_epoch", "B_RTN_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key.startswith("codice_hi"): - data = np.full((4 * n, 15, 4, 4), fillval, dtype=np.float32) - dims = [ - "codice_hi_epoch", - "codice_hi_energy_range", - "codice_hi_spin_sector", - "codice_hi_elevation", - ] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key == "swe_counterstreaming_electrons": - data = np.full(n, fillval, dtype=np.uint8) - dims = ["epoch"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key.startswith("swe"): - data = np.full((n, 8), fillval, dtype=np.uint32) - dims = ["epoch", "swe_electron_energy_labels"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - elif key.startswith("hit"): - data = np.full(n, fillval, dtype=np.uint32) - dims = ["epoch"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - else: - data = np.full(n, fillval, dtype=np.float32) - dims = ["epoch"] - dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - # Populate the dataset variables for i, record in enumerate(records): for key, val in record.items(): - if key in [ - "apid", - "met", - "met_in_utc", - "ttj2000ns", - "last_modified", - "mag_hk_status", - "spice_kernels", - "instrument", - ]: + attrs = cdf_manager.get_variable_attributes(key, check_schema=False) + if key not in IALIRT_KEYS: continue - elif key in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]: - dataset[key].data[i, :] = val - elif key.startswith("swe_normalized_counts"): - dataset[key].data[i, :] = val - elif key.startswith("codice_hi"): - dataset[key].data[i, :, :, :, :] = val + elif key == "mag_B_GSE": + dims = ["mag_epoch", "B_GSE_labels"] + elif key in ["sc_position_GSE", "sc_velocity_GSE"]: + dims = ["epoch", "sc_GSE_labels"] + elif key in ["sc_position_GSM", "sc_velocity_GSM"]: + dims = ["epoch", "sc_GSM_labels"] + elif key == "mag_B_GSE": + dims = ["mag_epoch", "B_GSE_labels"] + elif key == "mag_B_GSM": + dims = ["mag_epoch", "B_GSM_labels"] + elif key == "mag_B_RTN": + dims = ["mag_epoch", "B_RTN_labels"] + elif key.startswith("swe"): + dims = ["epoch", "swe_electron_energy_labels"] + elif key == "codice_hi_h": + dims = [ + "codice_hi_epoch", + "codice_hi_h_energy_range", + "codice_hi_spin_sector", + "codice_hi_elevation", + ] else: - dataset[key].data[i] = val + dims = [f"{record['instrument']}_epoch"] + + dataset[key] = xr.DataArray(val, dims=dims, attrs=attrs) return dataset diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 9a6de42371..c540b46225 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -18,37 +18,6 @@ def test_create_dataset(): "met_in_utc": "2025-06-20T08:00:00", "ttj2000ns": 123456789000000, "instrument": "codice_hi", - "swe_normalized_counts": [Decimal("0.0") for _ in range(8)], - "swe_counterstreaming_electrons": Decimal("0.0"), - "swapi_pseudo_proton_speed": Decimal("0.0"), - "swapi_pseudo_proton_density": Decimal("0.0"), - "swapi_pseudo_proton_temperature": Decimal("0.0"), - "hit_e_a_side_low_en": Decimal("0.0"), - "hit_e_a_side_med_en": Decimal("0.0"), - "hit_e_a_side_high_en": Decimal("0.0"), - "hit_e_b_side_low_en": Decimal("0.0"), - "hit_e_b_side_med_en": Decimal("0.0"), - "hit_e_b_side_high_en": Decimal("0.0"), - "hit_h_omni_med_en": Decimal("0.0"), - "hit_h_a_side_high_en": Decimal("0.0"), - "hit_h_b_side_high_en": Decimal("0.0"), - "hit_he_omni_low_en": Decimal("0.0"), - "hit_he_omni_high_en": Decimal("0.0"), - "mag_epoch": Decimal("0.0"), - "mag_B_GSE": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_GSM": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_RTN": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_magnitude": Decimal("0.0"), - "mag_phi_B_GSM": Decimal("0.0"), - "mag_theta_B_GSM": Decimal("0.0"), - "mag_phi_B_GSE": Decimal("0.0"), - "mag_theta_B_GSE": Decimal("0.0"), - "codice_lo_c_over_o_abundance": Decimal("0.0"), - "codice_lo_mg_over_o_abundance": Decimal("0.0"), - "codice_lo_fe_over_o_abundance": Decimal("0.0"), - "codice_lo_c_plus_6_over_c_plus_5_ratio": Decimal("0.0"), - "codice_lo_o_plus_7_over_o_plus_6_ratio": Decimal("0.0"), - "codice_lo_fe_low_over_fe_high_ratio": Decimal("0.0"), "codice_hi_h": [ [[Decimal("0.0") for _ in range(4)] for _ in range(4)] for _ in range(15) @@ -59,18 +28,6 @@ def test_create_dataset(): Decimal("0.0"), Decimal("0.0"), ], - "codice_hi_h_spin_angle": [ - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - ], - "codice_hi_h_elevation_angle": [ - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - ], }, { "apid": 478, @@ -98,6 +55,15 @@ def test_create_dataset(): "swapi_pseudo_proton_density": Decimal("0.0"), "swapi_pseudo_proton_temperature": Decimal("0.0"), }, + { + "apid": 478, + "met": 123456789, + "met_in_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000002, + "instrument": "swe", + "swe_normalized_counts": [Decimal("0.0") for _ in range(8)], + "swe_counterstreaming_electrons": Decimal("0.0"), + }, ] dataset = create_xarray_from_records(records) From b025c03368dd00ef1fe276672e924cc035f2ae62 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 13:11:06 -0700 Subject: [PATCH 19/42] add dimensions --- imap_processing/ialirt/utils/constants.py | 86 ++++++++++--------- imap_processing/ialirt/utils/create_xarray.py | 18 +++- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 7c50686dd4..4a2027dbac 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -1,73 +1,77 @@ """Keys for I-ALiRT data products.""" -IALIRT_KEYS = [ +# ---- the actual mapping ---- +IALIRT_DIMS = { # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins - "codice_hi_h", + "codice_hi_h": [ + "codice_hi_epoch", + "codice_hi_h_energy_range", + "codice_hi_spin_sector", + "codice_hi_elevation", + ], # C/O abundance ratio - "codice_lo_c_over_o_abundance", + "codice_lo_c_over_o_abundance": ["codice_lo_epoch"], # Mg/O abundance ratio - "codice_lo_mg_over_o_abundance", + "codice_lo_mg_over_o_abundance": ["codice_lo_epoch"], # Fe/O abundance ratio - "codice_lo_fe_over_o_abundance", + "codice_lo_fe_over_o_abundance": ["codice_lo_epoch"], # C+6/C+5 charge state ratio - "codice_lo_c_plus_6_over_c_plus_5_ratio", + "codice_lo_c_plus_6_over_c_plus_5_ratio": ["codice_lo_epoch"], # O+7/O+6 charge state ratio - "codice_lo_o_plus_7_over_o_plus_6_ratio", + "codice_lo_o_plus_7_over_o_plus_6_ratio": ["codice_lo_epoch"], # Fe low/Fe high charge state ratio - "codice_lo_fe_low_over_fe_high_ratio", + "codice_lo_fe_low_over_fe_high_ratio": ["codice_lo_epoch"], # Low energy (>0.5 MeV) electrons (A-side) - "hit_e_a_side_low_en", + "hit_e_a_side_low_en": ["hit_epoch"], # Medium energy (<1 MeV) electrons (A-side) - "hit_e_a_side_med_en", + "hit_e_a_side_med_en": ["hit_epoch"], # Low energy (>0.5 MeV) electrons (B-side) - "hit_e_b_side_low_en", + "hit_e_b_side_low_en": ["hit_epoch"], # Medium energy (<1 MeV) electrons (B-side) - "hit_e_b_side_med_en", + "hit_e_b_side_med_en": ["hit_epoch"], # Low energy (6 to 8 MeV) protons (Omnidirectional) - "hit_h_omni_low_en", + "hit_h_omni_low_en": ["hit_epoch"], # Medium energy (12 to 15 MeV) protons (Omnidirectional) - "hit_h_omni_med_en", + "hit_h_omni_med_en": ["hit_epoch"], # Low energy (6 to 8 MeV/nuc) He (Omnidirectional) - "hit_he_omni_low_en", + "hit_he_omni_low_en": ["hit_epoch"], # High energy (15 to 70 MeV/nuc) He (Omnidirectional) - "hit_he_omni_high_en", - # MAG instrument epoch - "mag_epoch", + "hit_he_omni_high_en": ["hit_epoch"], # Magnitude of the magnetic field vector - "mag_B_magnitude", - # Magnetic field vector in RTN coordinates - "mag_B_RTN", - # Magnetic field vector in GSE coordinates - "mag_B_GSE", + "mag_B_magnitude": ["mag_epoch"], # Elevation angle (θ) of the magnetic field in GSE coordinates - "mag_theta_B_GSE", + "mag_theta_B_GSE": ["mag_epoch"], # Azimuth angle (φ) of the magnetic field in GSE coordinates - "mag_phi_B_GSE", - # Magnetic field vector in GSM coordinates - "mag_B_GSM", + "mag_phi_B_GSE": ["mag_epoch"], # Elevation angle (θ) of the magnetic field in GSM coordinates - "mag_theta_B_GSM", + "mag_theta_B_GSM": ["mag_epoch"], # Azimuth angle (φ) of the magnetic field in GSM coordinates - "mag_phi_B_GSM", + "mag_phi_B_GSM": ["mag_epoch"], + # Magnetic field vector in GSE coordinates + "mag_B_GSE": ["mag_epoch", "B_GSE_labels"], + # Magnetic field vector in GSM coordinates + "mag_B_GSM": ["mag_epoch", "B_GSM_labels"], + # Magnetic field vector in RTN coordinates + "mag_B_RTN": ["mag_epoch", "B_RTN_labels"], # Pseudo density of solar wind protons - "swapi_pseudo_proton_density", + "swapi_pseudo_proton_density": ["swapi_epoch"], # Pseudo speed of solar wind protons in solar inertial frame - "swapi_pseudo_proton_speed", + "swapi_pseudo_proton_speed": ["swapi_epoch"], # Pseudo temperature of solar wind protons in plasma frame - "swapi_pseudo_proton_temperature", + "swapi_pseudo_proton_temperature": ["swapi_epoch"], # SWE Normalized Counts - "swe_normalized_counts", + "swe_normalized_counts": ["swe_epoch", "swe_electron_energy_labels"], # SWE Counterstreaming flag - "swe_counterstreaming_electrons", - # Spacecraft position in GSM coordinates - "sc_position_GSM", - # Spacecraft velocity in GSM coordinates - "sc_velocity_GSM", + "swe_counterstreaming_electrons": ["swe_epoch"], # Spacecraft position in GSE coordinates - "sc_position_GSE", + "sc_position_GSE": ["spacecraft_epoch", "sc_GSE_labels"], # Spacecraft velocity in GSE coordinates - "sc_velocity_GSE", -] + "sc_velocity_GSE": ["spacecraft_epoch", "sc_GSE_labels"], + # Spacecraft position in GSM coordinates + "sc_position_GSM": ["spacecraft_epoch", "sc_GSM_labels"], + # Spacecraft velocity in GSM coordinates + "sc_velocity_GSM": ["spacecraft_epoch", "sc_GSM_labels"], +} codice_energy_bounds = [ (0.020, 0.028), diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 0ae86bfaa4..8b7141ae18 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -9,7 +9,7 @@ HI_IALIRT_REF_SPIN_ANGLE, ) from imap_processing.ialirt.utils.constants import ( - IALIRT_KEYS, + IALIRT_DIMS, codice_energy_bounds, swe_energy_labels, ) @@ -169,6 +169,22 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"), ) + for key in IALIRT_DIMS.keys(): + dims = IALIRT_DIMS[key] + attrs = cdf_manager.get_variable_attributes(key, check_schema=False) + fill = attrs["FILLVAL"] + + shape = [dataset.dims[d] for d in dims] + + dtype = np.float32 + if key in { + "swe_counterstreaming_electrons", + }: + dtype = np.uint8 + + data = np.full(shape, fill, dtype=dtype) + dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) + # Populate the dataset variables for i, record in enumerate(records): for key, val in record.items(): From c8bc9a51257875c3f67a7b4911154c29e8d9289f Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 13:30:51 -0700 Subject: [PATCH 20/42] udpate all --- imap_processing/ialirt/utils/constants.py | 54 ++++++++- imap_processing/ialirt/utils/create_xarray.py | 110 +++++++++++------- 2 files changed, 124 insertions(+), 40 deletions(-) diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 4a2027dbac..8ae2b62fe7 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -1,11 +1,13 @@ """Keys for I-ALiRT data products.""" +import numpy as np + # ---- the actual mapping ---- IALIRT_DIMS = { # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins "codice_hi_h": [ "codice_hi_epoch", - "codice_hi_h_energy_range", + "codice_hi_h_energy_range_labels", "codice_hi_spin_sector", "codice_hi_elevation", ], @@ -73,6 +75,56 @@ "sc_velocity_GSM": ["spacecraft_epoch", "sc_GSM_labels"], } +IALIRT_DTYPES = { + # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins + "codice_hi_h": np.float32, + + # CoDICE-Lo abundance / ratios + "codice_lo_c_over_o_abundance": np.float32, + "codice_lo_mg_over_o_abundance": np.float32, + "codice_lo_fe_over_o_abundance": np.float32, + "codice_lo_c_plus_6_over_c_plus_5_ratio": np.float32, + "codice_lo_o_plus_7_over_o_plus_6_ratio": np.float32, + "codice_lo_fe_low_over_fe_high_ratio": np.float32, + + # HIT scalars + "hit_e_a_side_low_en": np.uint32, + "hit_e_a_side_med_en": np.uint32, + "hit_e_b_side_low_en": np.uint32, + "hit_e_b_side_med_en": np.uint32, + "hit_h_omni_low_en": np.uint32, + "hit_h_omni_med_en": np.uint32, + "hit_he_omni_low_en": np.uint32, + "hit_he_omni_high_en": np.uint32, + + # MAG + "mag_epoch": np.int64, # if you are treating this as a data variable + "mag_B_magnitude": np.float32, + "mag_B_RTN": np.float32, + "mag_B_GSE": np.float32, + "mag_theta_B_GSE": np.float32, + "mag_phi_B_GSE": np.float32, + "mag_B_GSM": np.float32, + "mag_theta_B_GSM": np.float32, + "mag_phi_B_GSM": np.float32, + + # SWAPI + "swapi_pseudo_proton_density": np.float32, + "swapi_pseudo_proton_speed": np.float32, + "swapi_pseudo_proton_temperature": np.float32, + + # SWE + "swe_normalized_counts": np.uint32, + "swe_counterstreaming_electrons": np.uint8, + + # Spacecraft vectors + "sc_position_GSM": np.float32, + "sc_velocity_GSM": np.float32, + "sc_position_GSE": np.float32, + "sc_velocity_GSE": np.float32, +} + + codice_energy_bounds = [ (0.020, 0.028), (0.028, 0.040), diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 8b7141ae18..e6b0ccaeb3 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -1,5 +1,6 @@ """Creates xarray based on structure of queried DynamoDB.""" +from collections import defaultdict import numpy as np import xarray as xr @@ -9,6 +10,7 @@ HI_IALIRT_REF_SPIN_ANGLE, ) from imap_processing.ialirt.utils.constants import ( + IALIRT_DTYPES, IALIRT_DIMS, codice_energy_bounds, swe_energy_labels, @@ -37,9 +39,11 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 MULTI_EPOCH = {"codice_hi"} epochs = {inst: [] for inst in (ONE_EPOCH | MULTI_EPOCH)} + by_inst: dict[str, list[dict]] = defaultdict(list) for r in records: inst = r.get("instrument") + by_inst[r["instrument"]].append(r) if inst in ONE_EPOCH: epochs[inst].append(r["mag_epoch"] if inst == "mag" else r["ttj2000ns"]) elif inst in MULTI_EPOCH: @@ -131,7 +135,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 spin_angle = xr.DataArray( data=spin_angles, name="codice_hi_spin_angle", - dims=["codice_hi_spin_sector", "codice_hi_elevation_angle"], + dims=["codice_hi_spin_sector", "codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( "codice_hi_spin_angle", check_schema=False ), @@ -159,7 +163,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "sc_GSM_labels": sc_gsm_component, "sc_GSE_labels": sc_gse_component, "swe_electron_energy_labels": swe_electron_energy_labels, - "codice_hi_h_energy_range": energy_range, + "codice_hi_h_energy_range_labels": energy_range, "codice_hi_elevation": elevation, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, @@ -173,48 +177,76 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dims = IALIRT_DIMS[key] attrs = cdf_manager.get_variable_attributes(key, check_schema=False) fill = attrs["FILLVAL"] + dtype = IALIRT_DTYPES[key] shape = [dataset.dims[d] for d in dims] - dtype = np.float32 - if key in { - "swe_counterstreaming_electrons", - }: - dtype = np.uint8 - data = np.full(shape, fill, dtype=dtype) dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - # Populate the dataset variables - for i, record in enumerate(records): - for key, val in record.items(): - attrs = cdf_manager.get_variable_attributes(key, check_schema=False) - if key not in IALIRT_KEYS: - continue - elif key == "mag_B_GSE": - dims = ["mag_epoch", "B_GSE_labels"] - elif key in ["sc_position_GSE", "sc_velocity_GSE"]: - dims = ["epoch", "sc_GSE_labels"] - elif key in ["sc_position_GSM", "sc_velocity_GSM"]: - dims = ["epoch", "sc_GSM_labels"] - elif key == "mag_B_GSE": - dims = ["mag_epoch", "B_GSE_labels"] - elif key == "mag_B_GSM": - dims = ["mag_epoch", "B_GSM_labels"] - elif key == "mag_B_RTN": - dims = ["mag_epoch", "B_RTN_labels"] - elif key.startswith("swe"): - dims = ["epoch", "swe_electron_energy_labels"] - elif key == "codice_hi_h": - dims = [ - "codice_hi_epoch", - "codice_hi_h_energy_range", - "codice_hi_spin_sector", - "codice_hi_elevation", - ] - else: - dims = [f"{record['instrument']}_epoch"] - - dataset[key] = xr.DataArray(val, dims=dims, attrs=attrs) + for i, r in enumerate(by_inst.get("mag", [])): + dataset["mag_B_GSE"].data[i, :] = np.asarray(r["mag_B_GSE"], dtype=np.float32) + dataset["mag_B_GSM"].data[i, :] = np.asarray(r["mag_B_GSM"], dtype=np.float32) + dataset["mag_B_RTN"].data[i, :] = np.asarray(r["mag_B_RTN"], dtype=np.float32) + + dataset["mag_B_magnitude"].data[i] = np.float32(r["mag_B_magnitude"]) + dataset["mag_theta_B_GSE"].data[i] = np.float32(r["mag_theta_B_GSE"]) + dataset["mag_phi_B_GSE"].data[i] = np.float32(r["mag_phi_B_GSE"]) + dataset["mag_theta_B_GSM"].data[i] = np.float32(r["mag_theta_B_GSM"]) + dataset["mag_phi_B_GSM"].data[i] = np.float32(r["mag_phi_B_GSM"]) + + for i, r in enumerate(by_inst.get("codice_hi", [])): + sl = slice(4 * i, 4 * i + 4) + + hi = np.asarray(r["codice_hi_h"], dtype=np.float32) + + # Allow either (4,15,4,4) or (15,4,4) + if hi.shape == (15, 4, 4): + hi = np.repeat(hi[None, ...], 4, axis=0) + + dataset["codice_hi_h"].data[sl, :, :, :] = hi + + CODICE_LO_KEYS = [ + "codice_lo_c_over_o_abundance", + "codice_lo_mg_over_o_abundance", + "codice_lo_fe_over_o_abundance", + "codice_lo_c_plus_6_over_c_plus_5_ratio", + "codice_lo_o_plus_7_over_o_plus_6_ratio", + "codice_lo_fe_low_over_fe_high_ratio", + ] + + for i, r in enumerate(by_inst.get("codice_lo", [])): + for k in CODICE_LO_KEYS: + dataset[k].data[i] = np.float32(r[k]) + + HIT_KEYS = [ + "hit_e_a_side_low_en", + "hit_e_a_side_med_en", + "hit_e_b_side_low_en", + "hit_e_b_side_med_en", + "hit_h_omni_low_en", + "hit_h_omni_med_en", + "hit_he_omni_low_en", + "hit_he_omni_high_en", + ] + + for i, r in enumerate(by_inst.get("hit", [])): + for k in HIT_KEYS: + dataset[k].data[i] = np.uint32(r[k]) + + for i, r in enumerate(by_inst.get("swapi", [])): + dataset["swapi_pseudo_proton_density"].data[i] = np.float32(r["swapi_pseudo_proton_density"]) + dataset["swapi_pseudo_proton_speed"].data[i] = np.float32(r["swapi_pseudo_proton_speed"]) + dataset["swapi_pseudo_proton_temperature"].data[i] = np.float32(r["swapi_pseudo_proton_temperature"]) + + for i, r in enumerate(by_inst.get("swe", [])): + dataset["swe_normalized_counts"].data[i, :] = np.asarray(r["swe_normalized_counts"], dtype=np.uint32) + dataset["swe_counterstreaming_electrons"].data[i] = np.uint8(r["swe_counterstreaming_electrons"]) + + for i, r in enumerate(by_inst.get("spacecraft", [])): + dataset["sc_position_GSE"].data[i, :] = np.asarray(r["sc_position_GSE"], dtype=np.float32) + dataset["sc_velocity_GSE"].data[i, :] = np.asarray(r["sc_velocity_GSE"], dtype=np.float32) + dataset["sc_position_GSM"].data[i, :] = np.asarray(r["sc_position_GSM"], dtype=np.float32) + dataset["sc_velocity_GSM"].data[i, :] = np.asarray(r["sc_velocity_GSM"], dtype=np.float32) return dataset From 2891967f675d7d52fbab7e49b9d814793705fb25 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 14:04:33 -0700 Subject: [PATCH 21/42] udpate all --- .../config/imap_ialirt_l1_variable_attrs.yaml | 6 +- imap_processing/ialirt/utils/constants.py | 20 ++-- imap_processing/ialirt/utils/create_xarray.py | 107 ++++++++---------- .../tests/ialirt/unit/test_create_xarray.py | 41 ++++++- 4 files changed, 97 insertions(+), 77 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 8802ca8dad..994ca3e576 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -243,7 +243,7 @@ codice_lo_fe_over_o_abundance: VALIDMAX: 100000000.0 FORMAT: F16.6 -codice_lo_c_plus_6_over_c_plus_5_ratio: +codice_lo_c_plus_6_over_c_plus_5: <<: *default_float32 CATDESC: C+6/C+5 charge state ratio FIELDNAM: C+6/C+5 charge state ratio @@ -254,7 +254,7 @@ codice_lo_c_plus_6_over_c_plus_5_ratio: VALIDMAX: 100000000.0 FORMAT: F16.6 -codice_lo_o_plus_7_over_o_plus_6_ratio: +codice_lo_o_plus_7_over_o_plus_6: <<: *default_float32 CATDESC: O+7/O+6 charge state ratio FIELDNAM: O7+/O6+ charge state ratio @@ -265,7 +265,7 @@ codice_lo_o_plus_7_over_o_plus_6_ratio: VALIDMAX: 100000000.0 FORMAT: F16.6 -codice_lo_fe_low_over_fe_high_ratio: +codice_lo_fe_low_over_fe_high: <<: *default_float32 CATDESC: Fe low/high charge state ratio (low=6–12, high=13–17) FIELDNAM: Fe low/high charge state ratio diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 8ae2b62fe7..051bf072c3 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -18,11 +18,11 @@ # Fe/O abundance ratio "codice_lo_fe_over_o_abundance": ["codice_lo_epoch"], # C+6/C+5 charge state ratio - "codice_lo_c_plus_6_over_c_plus_5_ratio": ["codice_lo_epoch"], + "codice_lo_c_plus_6_over_c_plus_5": ["codice_lo_epoch"], # O+7/O+6 charge state ratio - "codice_lo_o_plus_7_over_o_plus_6_ratio": ["codice_lo_epoch"], + "codice_lo_o_plus_7_over_o_plus_6": ["codice_lo_epoch"], # Fe low/Fe high charge state ratio - "codice_lo_fe_low_over_fe_high_ratio": ["codice_lo_epoch"], + "codice_lo_fe_low_over_fe_high": ["codice_lo_epoch"], # Low energy (>0.5 MeV) electrons (A-side) "hit_e_a_side_low_en": ["hit_epoch"], # Medium energy (<1 MeV) electrons (A-side) @@ -78,15 +78,13 @@ IALIRT_DTYPES = { # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins "codice_hi_h": np.float32, - # CoDICE-Lo abundance / ratios "codice_lo_c_over_o_abundance": np.float32, "codice_lo_mg_over_o_abundance": np.float32, "codice_lo_fe_over_o_abundance": np.float32, - "codice_lo_c_plus_6_over_c_plus_5_ratio": np.float32, - "codice_lo_o_plus_7_over_o_plus_6_ratio": np.float32, - "codice_lo_fe_low_over_fe_high_ratio": np.float32, - + "codice_lo_c_plus_6_over_c_plus_5": np.float32, + "codice_lo_o_plus_7_over_o_plus_6": np.float32, + "codice_lo_fe_low_over_fe_high": np.float32, # HIT scalars "hit_e_a_side_low_en": np.uint32, "hit_e_a_side_med_en": np.uint32, @@ -96,9 +94,8 @@ "hit_h_omni_med_en": np.uint32, "hit_he_omni_low_en": np.uint32, "hit_he_omni_high_en": np.uint32, - # MAG - "mag_epoch": np.int64, # if you are treating this as a data variable + "mag_epoch": np.int64, # if you are treating this as a data variable "mag_B_magnitude": np.float32, "mag_B_RTN": np.float32, "mag_B_GSE": np.float32, @@ -107,16 +104,13 @@ "mag_B_GSM": np.float32, "mag_theta_B_GSM": np.float32, "mag_phi_B_GSM": np.float32, - # SWAPI "swapi_pseudo_proton_density": np.float32, "swapi_pseudo_proton_speed": np.float32, "swapi_pseudo_proton_temperature": np.float32, - # SWE "swe_normalized_counts": np.uint32, "swe_counterstreaming_electrons": np.uint8, - # Spacecraft vectors "sc_position_GSM": np.float32, "sc_velocity_GSM": np.float32, diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index e6b0ccaeb3..f7c5b568b7 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -1,6 +1,7 @@ """Creates xarray based on structure of queried DynamoDB.""" from collections import defaultdict + import numpy as np import xarray as xr @@ -10,8 +11,8 @@ HI_IALIRT_REF_SPIN_ANGLE, ) from imap_processing.ialirt.utils.constants import ( - IALIRT_DTYPES, IALIRT_DIMS, + IALIRT_DTYPES, codice_energy_bounds, swe_energy_labels, ) @@ -35,18 +36,18 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 cdf_manager.add_instrument_global_attrs("ialirt") cdf_manager.add_instrument_variable_attrs("ialirt", "l1") - ONE_EPOCH = {"codice_lo", "hit", "swapi", "swe", "spacecraft", "mag"} - MULTI_EPOCH = {"codice_hi"} + one_epoch = {"codice_lo", "hit", "swapi", "swe", "spacecraft", "mag"} + multi_epoch = {"codice_hi"} - epochs = {inst: [] for inst in (ONE_EPOCH | MULTI_EPOCH)} + epochs: dict[str, list[int]] = {inst: [] for inst in (one_epoch | multi_epoch)} by_inst: dict[str, list[dict]] = defaultdict(list) for r in records: inst = r.get("instrument") by_inst[r["instrument"]].append(r) - if inst in ONE_EPOCH: + if inst in one_epoch: epochs[inst].append(r["mag_epoch"] if inst == "mag" else r["ttj2000ns"]) - elif inst in MULTI_EPOCH: + elif inst in multi_epoch: epochs[inst].extend(r["codice_hi_epoch"]) epoch_arrays = {} @@ -117,7 +118,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 HI_IALIRT_ELEVATION_ANGLE, name="codice_hi_elevation", dims=["codice_hi_elevation"], - attrs=cdf_manager.get_variable_attributes("codice_hi_elevation", check_schema=False), + attrs=cdf_manager.get_variable_attributes( + "codice_hi_elevation", check_schema=False + ), ) # Calculate spin angle for CoDICE-Hi @@ -155,6 +158,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "codice_lo_epoch": epoch_arrays["codice_lo_epoch"], "hit_epoch": epoch_arrays["hit_epoch"], "mag_epoch": epoch_arrays["mag_epoch"], + "spacecraft_epoch": epoch_arrays["spacecraft_epoch"], "swapi_epoch": epoch_arrays["swapi_epoch"], "swe_epoch": epoch_arrays["swe_epoch"], "B_GSM_labels": gsm_component, @@ -173,6 +177,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"), ) + # Create variables with fill values. for key in IALIRT_DIMS.keys(): dims = IALIRT_DIMS[key] attrs = cdf_manager.get_variable_attributes(key, check_schema=False) @@ -185,68 +190,52 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) for i, r in enumerate(by_inst.get("mag", [])): - dataset["mag_B_GSE"].data[i, :] = np.asarray(r["mag_B_GSE"], dtype=np.float32) - dataset["mag_B_GSM"].data[i, :] = np.asarray(r["mag_B_GSM"], dtype=np.float32) - dataset["mag_B_RTN"].data[i, :] = np.asarray(r["mag_B_RTN"], dtype=np.float32) - - dataset["mag_B_magnitude"].data[i] = np.float32(r["mag_B_magnitude"]) - dataset["mag_theta_B_GSE"].data[i] = np.float32(r["mag_theta_B_GSE"]) - dataset["mag_phi_B_GSE"].data[i] = np.float32(r["mag_phi_B_GSE"]) - dataset["mag_theta_B_GSM"].data[i] = np.float32(r["mag_theta_B_GSM"]) - dataset["mag_phi_B_GSM"].data[i] = np.float32(r["mag_phi_B_GSM"]) - - for i, r in enumerate(by_inst.get("codice_hi", [])): - sl = slice(4 * i, 4 * i + 4) - + for k in IALIRT_DIMS.keys(): + if k in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]: + dataset[k].data[i, :] = np.asarray(r[k], dtype=np.float32) + + if k in [ + "mag_B_magnitude", + "mag_theta_B_GSE", + "mag_phi_B_GSE", + "mag_theta_B_GSM", + "mag_phi_B_GSM", + ]: + dataset[k].data[i] = np.float32(r[k]) + + for rec_i, r in enumerate(by_inst.get("codice_hi", [])): + # 4 high epochs per record + t0 = 4 * rec_i + t1 = t0 + 4 hi = np.asarray(r["codice_hi_h"], dtype=np.float32) - - # Allow either (4,15,4,4) or (15,4,4) - if hi.shape == (15, 4, 4): - hi = np.repeat(hi[None, ...], 4, axis=0) - - dataset["codice_hi_h"].data[sl, :, :, :] = hi - - CODICE_LO_KEYS = [ - "codice_lo_c_over_o_abundance", - "codice_lo_mg_over_o_abundance", - "codice_lo_fe_over_o_abundance", - "codice_lo_c_plus_6_over_c_plus_5_ratio", - "codice_lo_o_plus_7_over_o_plus_6_ratio", - "codice_lo_fe_low_over_fe_high_ratio", - ] + dataset["codice_hi_h"].data[t0:t1, :, :, :] = hi for i, r in enumerate(by_inst.get("codice_lo", [])): - for k in CODICE_LO_KEYS: - dataset[k].data[i] = np.float32(r[k]) - - HIT_KEYS = [ - "hit_e_a_side_low_en", - "hit_e_a_side_med_en", - "hit_e_b_side_low_en", - "hit_e_b_side_med_en", - "hit_h_omni_low_en", - "hit_h_omni_med_en", - "hit_he_omni_low_en", - "hit_he_omni_high_en", - ] + for k in IALIRT_DIMS.keys(): + if k.startswith("codice_lo_"): + dataset[k].data[i] = np.float32(r[k]) for i, r in enumerate(by_inst.get("hit", [])): - for k in HIT_KEYS: - dataset[k].data[i] = np.uint32(r[k]) + for k in IALIRT_DIMS.keys(): + if k.startswith("hit_"): + dataset[k].data[i] = np.uint32(r[k]) for i, r in enumerate(by_inst.get("swapi", [])): - dataset["swapi_pseudo_proton_density"].data[i] = np.float32(r["swapi_pseudo_proton_density"]) - dataset["swapi_pseudo_proton_speed"].data[i] = np.float32(r["swapi_pseudo_proton_speed"]) - dataset["swapi_pseudo_proton_temperature"].data[i] = np.float32(r["swapi_pseudo_proton_temperature"]) + for k in IALIRT_DIMS.keys(): + if k.startswith("swapi_"): + dataset[k].data[i] = np.float32(r[k]) for i, r in enumerate(by_inst.get("swe", [])): - dataset["swe_normalized_counts"].data[i, :] = np.asarray(r["swe_normalized_counts"], dtype=np.uint32) - dataset["swe_counterstreaming_electrons"].data[i] = np.uint8(r["swe_counterstreaming_electrons"]) + dataset["swe_normalized_counts"].data[i, :] = np.asarray( + r["swe_normalized_counts"], dtype=np.uint32 + ) + dataset["swe_counterstreaming_electrons"].data[i] = np.uint8( + r["swe_counterstreaming_electrons"] + ) for i, r in enumerate(by_inst.get("spacecraft", [])): - dataset["sc_position_GSE"].data[i, :] = np.asarray(r["sc_position_GSE"], dtype=np.float32) - dataset["sc_velocity_GSE"].data[i, :] = np.asarray(r["sc_velocity_GSE"], dtype=np.float32) - dataset["sc_position_GSM"].data[i, :] = np.asarray(r["sc_position_GSM"], dtype=np.float32) - dataset["sc_velocity_GSM"].data[i, :] = np.asarray(r["sc_velocity_GSM"], dtype=np.float32) + for k in IALIRT_DIMS.keys(): + if k.startswith("sc_"): + dataset[k].data[i, :] = np.asarray(r[k], dtype=np.float32) return dataset diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index c540b46225..a101e9b493 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -19,8 +19,11 @@ def test_create_dataset(): "ttj2000ns": 123456789000000, "instrument": "codice_hi", "codice_hi_h": [ - [[Decimal("0.0") for _ in range(4)] for _ in range(4)] - for _ in range(15) + [ + [[Decimal("0.0") for _ in range(4)] for _ in range(4)] + for _ in range(15) + ] + for _ in range(4) ], "codice_hi_epoch": [ Decimal("0.0"), @@ -29,6 +32,40 @@ def test_create_dataset(): Decimal("0.0"), ], }, + { + "apid": 478, + "met": 123456789, + "met_in_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000000, + "instrument": "codice_hi", + "codice_hi_h": [ + [ + [[Decimal("0.0") for _ in range(4)] for _ in range(4)] + for _ in range(15) + ] + for _ in range(4) + ], + "codice_hi_epoch": [ + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + Decimal("0.0"), + ], + }, + { + "apid": 478, + "met": 123456789, + "met_in_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000000, + "instrument": "codice_lo", + "codice_hi_epoch": Decimal("0.0"), + "codice_lo_c_over_o_abundance": Decimal("0.0"), + "codice_lo_mg_over_o_abundance": Decimal("0.0"), + "codice_lo_fe_over_o_abundance": Decimal("0.0"), + "codice_lo_c_plus_6_over_c_plus_5": Decimal("0.0"), + "codice_lo_o_plus_7_over_o_plus_6": Decimal("0.0"), + "codice_lo_fe_low_over_fe_high": Decimal("0.0"), + }, { "apid": 478, "met": 123456789, From 72113d723371a88fc76f0e385582bc0fa6762fd9 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 14:09:07 -0700 Subject: [PATCH 22/42] udpate all --- imap_processing/ialirt/utils/create_xarray.py | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index f7c5b568b7..784e23dc8a 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -42,13 +42,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 epochs: dict[str, list[int]] = {inst: [] for inst in (one_epoch | multi_epoch)} by_inst: dict[str, list[dict]] = defaultdict(list) - for r in records: - inst = r.get("instrument") - by_inst[r["instrument"]].append(r) + for record in records: + inst = record.get("instrument") + by_inst[record["instrument"]].append(record) if inst in one_epoch: - epochs[inst].append(r["mag_epoch"] if inst == "mag" else r["ttj2000ns"]) + epochs[inst].append( + record["mag_epoch"] if inst == "mag" else record["ttj2000ns"] + ) elif inst in multi_epoch: - epochs[inst].extend(r["codice_hi_epoch"]) + epochs[inst].extend(record["codice_hi_epoch"]) epoch_arrays = {} @@ -189,53 +191,53 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 data = np.full(shape, fill, dtype=dtype) dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) - for i, r in enumerate(by_inst.get("mag", [])): - for k in IALIRT_DIMS.keys(): - if k in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]: - dataset[k].data[i, :] = np.asarray(r[k], dtype=np.float32) + for i, record in enumerate(by_inst.get("mag", [])): + for key in IALIRT_DIMS.keys(): + if key in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]: + dataset[key].data[i, :] = np.asarray(record[key], dtype=np.float32) - if k in [ + if key in [ "mag_B_magnitude", "mag_theta_B_GSE", "mag_phi_B_GSE", "mag_theta_B_GSM", "mag_phi_B_GSM", ]: - dataset[k].data[i] = np.float32(r[k]) + dataset[key].data[i] = np.float32(record[key]) - for rec_i, r in enumerate(by_inst.get("codice_hi", [])): + for i, record in enumerate(by_inst.get("codice_hi", [])): # 4 high epochs per record - t0 = 4 * rec_i + t0 = 4 * i t1 = t0 + 4 - hi = np.asarray(r["codice_hi_h"], dtype=np.float32) + hi = np.asarray(record["codice_hi_h"], dtype=np.float32) dataset["codice_hi_h"].data[t0:t1, :, :, :] = hi - for i, r in enumerate(by_inst.get("codice_lo", [])): - for k in IALIRT_DIMS.keys(): - if k.startswith("codice_lo_"): - dataset[k].data[i] = np.float32(r[k]) + for i, record in enumerate(by_inst.get("codice_lo", [])): + for key in IALIRT_DIMS.keys(): + if key.startswith("codice_lo_"): + dataset[key].data[i] = np.float32(record[key]) - for i, r in enumerate(by_inst.get("hit", [])): - for k in IALIRT_DIMS.keys(): - if k.startswith("hit_"): - dataset[k].data[i] = np.uint32(r[k]) + for i, record in enumerate(by_inst.get("hit", [])): + for key in IALIRT_DIMS.keys(): + if key.startswith("hit_"): + dataset[key].data[i] = np.uint32(record[key]) - for i, r in enumerate(by_inst.get("swapi", [])): - for k in IALIRT_DIMS.keys(): - if k.startswith("swapi_"): - dataset[k].data[i] = np.float32(r[k]) + for i, record in enumerate(by_inst.get("swapi", [])): + for key in IALIRT_DIMS.keys(): + if key.startswith("swapi_"): + dataset[key].data[i] = np.float32(record[key]) - for i, r in enumerate(by_inst.get("swe", [])): + for i, record in enumerate(by_inst.get("swe", [])): dataset["swe_normalized_counts"].data[i, :] = np.asarray( - r["swe_normalized_counts"], dtype=np.uint32 + record["swe_normalized_counts"], dtype=np.uint32 ) dataset["swe_counterstreaming_electrons"].data[i] = np.uint8( - r["swe_counterstreaming_electrons"] + record["swe_counterstreaming_electrons"] ) - for i, r in enumerate(by_inst.get("spacecraft", [])): - for k in IALIRT_DIMS.keys(): - if k.startswith("sc_"): - dataset[k].data[i, :] = np.asarray(r[k], dtype=np.float32) + for i, record in enumerate(by_inst.get("spacecraft", [])): + for key in IALIRT_DIMS.keys(): + if key.startswith("sc_"): + dataset[key].data[i, :] = np.asarray(record[key], dtype=np.float32) return dataset From dccfeebeefe7de8dea9a5e65e008cdca23ae45b8 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 8 Jan 2026 16:19:02 -0700 Subject: [PATCH 23/42] update to epoch --- .../config/imap_ialirt_l1_variable_attrs.yaml | 135 ++++++++++++------ imap_processing/ialirt/utils/constants.py | 88 ++++++++---- imap_processing/ialirt/utils/create_xarray.py | 73 ++++++---- .../tests/ialirt/unit/test_create_xarray.py | 2 +- 4 files changed, 198 insertions(+), 100 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 994ca3e576..3603f08527 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -9,6 +9,59 @@ default_attrs: &default VAR_TYPE: data UNITS: " " +instrument_epoch: &instrument_epoch + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) + VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + RESOLUTION: ' ' + DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" + +codice_hi_epoch: + <<: *instrument_epoch + CATDESC: CoDICE-Hi time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: CoDICE-Hi epoch + +codice_lo_epoch: + <<: *instrument_epoch + CATDESC: CoDICE-Lo time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: CoDICE-Lo epoch + +hit_epoch: + <<: *instrument_epoch + CATDESC: HIT time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: HIT epoch + +mag_epoch: + <<: *instrument_epoch + CATDESC: MAG time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: MAG epoch + +swapi_epoch: + <<: *instrument_epoch + CATDESC: SWAPI time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: SWAPI epoch + +swe_epoch: + <<: *instrument_epoch + CATDESC: SWE time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: SWE epoch + +spacecraft_epoch: + <<: *instrument_epoch + CATDESC: Spacecraft time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: Spacecraft epoch + B_GSM_labels: CATDESC: B GSM component labels FIELDNAM: B GSM component labels @@ -117,12 +170,34 @@ default_float64_attrs: &default_float64 VALIDMAX: 1.7976931348623157e+308 dtype: float64 -codice_hi_h_energy_range_labels: - CATDESC: CoDICE-Hi proton energy bin labels (MeV) - FIELDNAM: CoDICE-Hi proton energy bin labels - FORMAT: A20 +codice_hi_energy_minus_delta: + <<: *default_float32 + CATDESC: CoDICE-Hi proton energy bin lower delta (MeV) + FIELDNAM: CoDICE-Hi proton energy bin lower delta + VAR_TYPE: support_data + UNITS: "MeV" + VALIDMAX: 200.0 + VALIDMIN: 0.05000000074505806 + DEPEND_1: codice_hi_energy_center + +codice_hi_energy_plus_delta: + <<: *default_float32 + CATDESC: CoDICE-Hi proton energy bin upper delta (MeV) + FIELDNAM: CoDICE-Hi proton energy bin upper delta + VAR_TYPE: support_data + UNITS: "MeV" + VALIDMAX: 200.0 + VALIDMIN: 0.05000000074505806 + DEPEND_1: codice_hi_energy_center + +codice_hi_energy_center: + <<: *default_float32 + CATDESC: CoDICE-Hi proton energy bin (MeV) + FIELDNAM: CoDICE-Hi proton energy bin VAR_TYPE: support_data UNITS: "MeV" + DELTA_PLUS_VAR: codice_hi_energy_plus_delta + DELTA_MINUS_VAR: codice_hi_energy_minus_delta codice_hi_spin_sector: CATDESC: CoDICE-Hi spin sector index @@ -136,6 +211,13 @@ codice_hi_spin_sector: VALIDMAX: 3 VAR_TYPE: support_data +swe_electron_energy: + <<: *default_float32 + CATDESC: SWE electron channels (nominal central energies) + FIELDNAM: SWE electron channels + VAR_TYPE: support_data + UNITS: "eV" + codice_hi_elevation: <<: *default_float32 CATDESC: Elevation Angle @@ -175,24 +257,6 @@ codice_hi_spin_angle: LABL_PTR_1: codice_hi_spin_sector_labels LABL_PTR_2: codice_hi_elevation_labels -codice_hi_epoch: - CATDESC: CoDICE-Hi instrument epoch time tag - FIELDNAM: CoDICE-Hi epoch - LABLAXIS: CoDICE-Hi epoch - FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf - VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) - VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid - RESOLUTION: ' ' - DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" - codice_hi_h: <<: *default_float32 CATDESC: Proton intensity in 4 spin angles, 15 energy ranges, 4 spin sectors, and 4 elevations @@ -202,10 +266,9 @@ codice_hi_h: UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 - LABL_PTR_1: codice_hi_h_energy_range_labels - DEPEND_2: codice_hi_h_spin_sector_labels + DEPEND_1: codice_hi_h_energy_range + DEPEND_2: codice_hi_h_spin_sector DEPEND_3: codice_hi_h_elevation - LABL_PTR_3: codice_hi_h_elevation_labels DISPLAY_TYPE: no_plot FORMAT: F16.6 VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. @@ -396,24 +459,6 @@ hit_he_omni_high_en: VALIDMAX: 1000000000 VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. -mag_epoch: - CATDESC: MAG instrument epoch time tag - FIELDNAM: MAG epoch - LABLAXIS: MAG epoch - FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf - VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) - VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid - RESOLUTION: ' ' - DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" - mag_B_GSE: <<: *default_float32 CATDESC: Magnetic field vector in GSE coordinates @@ -578,7 +623,9 @@ swe_normalized_counts: UNITS: Normalized counts VALIDMIN: 0 VALIDMAX: 500000 - LABL_PTR_1: swe_electron_energy_labels + DEPEND_1: swe_electron_energy_labels + DISPLAY_TYPE: stack_plot + LABLAXIS: Electrons VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. swe_counterstreaming_electrons: diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 051bf072c3..e60fe9de43 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -2,7 +2,6 @@ import numpy as np -# ---- the actual mapping ---- IALIRT_DIMS = { # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins "codice_hi_h": [ @@ -62,7 +61,7 @@ # Pseudo temperature of solar wind protons in plasma frame "swapi_pseudo_proton_temperature": ["swapi_epoch"], # SWE Normalized Counts - "swe_normalized_counts": ["swe_epoch", "swe_electron_energy_labels"], + "swe_normalized_counts": ["swe_epoch", "swe_electron_energy"], # SWE Counterstreaming flag "swe_counterstreaming_electrons": ["swe_epoch"], # Spacecraft position in GSE coordinates @@ -118,32 +117,65 @@ "sc_velocity_GSE": np.float32, } - -codice_energy_bounds = [ - (0.020, 0.028), - (0.028, 0.040), - (0.040, 0.057), - (0.057, 0.080), - (0.080, 0.113), - (0.113, 0.160), - (0.160, 0.226), - (0.226, 0.320), - (0.320, 0.453), - (0.453, 0.640), - (0.640, 0.905), - (0.905, 1.280), - (1.280, 1.810), - (1.810, 2.560), - (2.560, 3.620), +codice_hi_energy_centers = [ + 0.02378414, + 0.03363586, + 0.04756828, + 0.06727171, + 0.09513657, + 0.13454343, + 0.19027314, + 0.26908685, + 0.38054628, + 0.53817371, + 0.76109255, + 1.07634741, + 1.52218511, + 2.15269482, + 3.04437021, +] +codice_hi_energy_minus = [ + 0.00378414, + 0.00535159, + 0.00756828, + 0.01070317, + 0.01513657, + 0.02140634, + 0.03027314, + 0.04281268, + 0.06054628, + 0.08562537, + 0.12109255, + 0.17125073, + 0.24218511, + 0.34250146, + 0.48437021, +] +codice_hi_energy_plus = [ + 0.00450013, + 0.00636414, + 0.00900026, + 0.01272829, + 0.01800052, + 0.02545657, + 0.03600103, + 0.05091315, + 0.07200206, + 0.10182629, + 0.14400413, + 0.20365259, + 0.28800825, + 0.40730518, + 0.57601651, ] -swe_energy_labels = [ - "100.4", - "140.0", - "194.0", - "270.0", - "376.0", - "523.0", - "727.0", - "1011.0", +swe_energy = [ + 100.4, + 140.0, + 194.0, + 270.0, + 376.0, + 523.0, + 727.0, + 1011.0, ] diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 784e23dc8a..a149e87821 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -13,8 +13,10 @@ from imap_processing.ialirt.utils.constants import ( IALIRT_DIMS, IALIRT_DTYPES, - codice_energy_bounds, - swe_energy_labels, + codice_hi_energy_centers, + codice_hi_energy_minus, + codice_hi_energy_plus, + swe_energy, ) @@ -46,11 +48,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 inst = record.get("instrument") by_inst[record["instrument"]].append(record) if inst in one_epoch: - epochs[inst].append( - record["mag_epoch"] if inst == "mag" else record["ttj2000ns"] - ) + epochs[inst].append(record[f"{inst}_epoch"]) elif inst in multi_epoch: - epochs[inst].extend(record["codice_hi_epoch"]) + epochs[inst].extend(record[f"{inst}_epoch"]) epoch_arrays = {} @@ -60,7 +60,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 data=arr, name=coord, dims=[coord], - attrs=cdf_manager.get_variable_attributes(coord, check_schema=False), + attrs=cdf_manager.get_variable_attributes( + f"{inst}_epoch", check_schema=False + ), ) sc_gsm_component = xr.DataArray( @@ -98,21 +100,39 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("B_RTN_labels", check_schema=False), ) - swe_electron_energy_labels = xr.DataArray( - data=[f"{label} eV" for label in swe_energy_labels], - name="swe_electron_energy_labels", - dims=["swe_electron_energy_labels"], + swe_electron_energy = xr.DataArray( + data=swe_energy, + name="swe_electron_energy", + dims=["swe_electron_energy"], attrs=cdf_manager.get_variable_attributes( - "swe_electron_energy_labels", check_schema=False + "swe_electron_energy", check_schema=False ), ) - energy_range = xr.DataArray( - data=[f"{low:.4f}-{high:.4f} MeV" for low, high in codice_energy_bounds], - name="codice_hi_h_energy_range_labels", - dims=["codice_hi_h_energy_range_labels"], + codice_hi_energy_center = xr.DataArray( + data=codice_hi_energy_centers, + name="codice_hi_energy_center", + dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_energy_range_labels", check_schema=False + "codice_hi_energy_center", check_schema=False + ), + ) + + codice_energy_minus = xr.DataArray( + data=codice_hi_energy_minus, + name="codice_hi_energy_minus", + dims=["codice_hi_energy_center"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_energy_minus", check_schema=False + ), + ) + + codice_energy_plus = xr.DataArray( + data=codice_hi_energy_plus, + name="codice_hi_energy_plus", + dims=["codice_hi_energy_center"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_energy_plus", check_schema=False ), ) @@ -125,15 +145,12 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - # Calculate spin angle for CoDICE-Hi + # Calculate spin angle # Formula: # θ_(g,n) = (θ_(g,0)+90°* n) mod 360° # where # n is number of sectored angles, 0 to 3, # g is size of the group (inst_az), 0 to 3, - # Calculate spin angle by adding a base angle from L2_HI_SECTORED_ANGLE - # for each SSD index and then adding multiple of 30 degrees for each elevation. - # Then mod by 360 to keep it within 0-360 range. spin_angles = ( HI_IALIRT_REF_SPIN_ANGLE[:, np.newaxis] + np.array([0, 1, 2, 3]) * 90 ) % 360.0 @@ -148,10 +165,10 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 spin_sector = xr.DataArray( data=np.arange(4, dtype=np.uint8), - name="codice_hi_h_spin_sector", - dims=["codice_hi_h_spin_sector"], + name="codice_hi_spin_sector", + dims=["codice_hi_spin_sector"], attrs=cdf_manager.get_variable_attributes( - "codice_hi_h_spin_sector", check_schema=False + "codice_hi_spin_sector", check_schema=False ), ) @@ -168,8 +185,10 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "B_RTN_labels": rtn_component, "sc_GSM_labels": sc_gsm_component, "sc_GSE_labels": sc_gse_component, - "swe_electron_energy_labels": swe_electron_energy_labels, - "codice_hi_h_energy_range_labels": energy_range, + "swe_electron_energy": swe_electron_energy, + "codice_hi_energy_center": codice_hi_energy_center, + "codice_hi_energy_minus_delta": codice_energy_minus, + "codice_hi_energy_plus_delta": codice_energy_plus, "codice_hi_elevation": elevation, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, @@ -206,7 +225,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 dataset[key].data[i] = np.float32(record[key]) for i, record in enumerate(by_inst.get("codice_hi", [])): - # 4 high epochs per record + # 4 codice-hi epochs per record t0 = 4 * i t1 = t0 + 4 hi = np.asarray(record["codice_hi_h"], dtype=np.float32) diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index a101e9b493..85b7fef9b9 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -140,7 +140,7 @@ def test_create_dataset(): assert dataset["mag_B_GSE"].dims == ("epoch", "component") assert dataset["swe_normalized_counts"].dims == ("epoch", "esa_step") assert dataset["codice_hi_h"].dims == ( - "epoch", + "codice_hi_epoch", "codice_hi_h_energy_ranges", "codice_hi_h_elevation", "codice_hi_h_spin_angle", From 00e3bf75d0b3b2cbde41d85cc4c77394bfcca567 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 11:43:42 -0700 Subject: [PATCH 24/42] fixed tests --- .../config/imap_ialirt_l1_variable_attrs.yaml | 388 +++++++++--------- imap_processing/ialirt/utils/constants.py | 19 +- imap_processing/ialirt/utils/create_xarray.py | 45 +- .../tests/ialirt/unit/test_create_xarray.py | 198 +++++---- 4 files changed, 328 insertions(+), 322 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 3603f08527..79d19beacf 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -1,3 +1,4 @@ +# Defaults default_attrs: &default # Assumed values for all variable attrs unless overwritten DEPEND_0: epoch @@ -9,6 +10,63 @@ default_attrs: &default VAR_TYPE: data UNITS: " " +default_uint8_attrs: &default_uint8 + <<: *default + FILLVAL: 255 + FORMAT: I3 + VALIDMIN: 0 + VALIDMAX: 255 + dtype: uint8 + +default_uint16_attrs: &default_uint16 + <<: *default + FILLVAL: 65535 + FORMAT: I5 + VALIDMIN: 0 + VALIDMAX: 65535 + dtype: uint16 + +default_uint32_attrs: &default_uint32 + <<: *default + FILLVAL: 4294967295 + FORMAT: I10 + VALIDMIN: 0 + VALIDMAX: 4294967295 + dtype: uint32 + +default_int64_attrs: &default_int64 + <<: *default + FILLVAL: -9223372036854775808 + FORMAT: I20 + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + dtype: int64 + +default_int32_attrs: &default_int32 + <<: *default + FILLVAL: -2147483648 + FORMAT: I10 + VALIDMIN: -2147483648 + VALIDMAX: 2147483647 + dtype: int32 + +default_float32_attrs: &default_float32 + <<: *default + FILLVAL: -1.0e31 + FORMAT: F12.6 + VALIDMIN: -3.4028235e+38 + VALIDMAX: 3.4028235e+38 + dtype: float32 + +default_float64_attrs: &default_float64 + <<: *default + FILLVAL: -1.0e31 + FORMAT: F10.2 + VALIDMIN: -1.7976931348623157e+308 + VALIDMAX: 1.7976931348623157e+308 + dtype: float64 + +# Epochs instrument_epoch: &instrument_epoch CATDESC: Time, number of nanoseconds since J2000 with leap seconds included FIELDNAM: epoch @@ -57,11 +115,12 @@ swe_epoch: CATDESC: SWE time, number of nanoseconds since J2000 with leap seconds included FIELDNAM: SWE epoch -spacecraft_epoch: +epoch: <<: *instrument_epoch CATDESC: Spacecraft time, number of nanoseconds since J2000 with leap seconds included FIELDNAM: Spacecraft epoch +# Labels B_GSM_labels: CATDESC: B GSM component labels FIELDNAM: B GSM component labels @@ -97,78 +156,17 @@ sc_GSE_labels: VAR_TYPE: support_data UNITS: " " -esa_step: - CATDESC: ESA step number. The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. - FIELDNAM: ESA step index - LABLAXIS: ESA step - FORMAT: I2 - VALIDMIN: 0 - VALIDMAX: 7 - VAR_TYPE: metadata - UNITS: " " - -swe_electron_energy_labels: - CATDESC: SWE electron channel labels (nominal central energies) - FIELDNAM: SWE electron channel labels - FORMAT: A12 +# Coordinate Variables +codice_hi_energy_center: + <<: *default_float32 + CATDESC: CoDICE-Hi proton energy bin (MeV) + FIELDNAM: CoDICE-Hi proton energy bin VAR_TYPE: support_data - UNITS: " " - -default_uint8_attrs: &default_uint8 - <<: *default - FILLVAL: 255 - FORMAT: I3 - VALIDMIN: 0 - VALIDMAX: 255 - dtype: uint8 - -default_uint16_attrs: &default_uint16 - <<: *default - FILLVAL: 65535 - FORMAT: I5 - VALIDMIN: 0 - VALIDMAX: 65535 - dtype: uint16 - -default_uint32_attrs: &default_uint32 - <<: *default - FILLVAL: 4294967295 - FORMAT: I10 - VALIDMIN: 0 - VALIDMAX: 4294967295 - dtype: uint32 - -default_int64_attrs: &default_int64 - <<: *default - FILLVAL: -9223372036854775808 - FORMAT: I20 - VALIDMIN: -9223372036854775808 - VALIDMAX: 9223372036854775807 - dtype: int64 - -default_int32_attrs: &default_int32 - <<: *default - FILLVAL: -2147483648 - FORMAT: I10 - VALIDMIN: -2147483648 - VALIDMAX: 2147483647 - dtype: int32 - -default_float32_attrs: &default_float32 - <<: *default - FILLVAL: -1.0e31 - FORMAT: F12.6 - VALIDMIN: -3.4028235e+38 - VALIDMAX: 3.4028235e+38 - dtype: float32 - -default_float64_attrs: &default_float64 - <<: *default - FILLVAL: -1.0e31 - FORMAT: F10.2 - VALIDMIN: -1.7976931348623157e+308 - VALIDMAX: 1.7976931348623157e+308 - dtype: float64 + UNITS: "MeV" + VALIDMAX: 20.0 + VALIDMIN: 0.0 + DELTA_PLUS_VAR: codice_hi_energy_plus_delta + DELTA_MINUS_VAR: codice_hi_energy_minus_delta codice_hi_energy_minus_delta: <<: *default_float32 @@ -176,8 +174,8 @@ codice_hi_energy_minus_delta: FIELDNAM: CoDICE-Hi proton energy bin lower delta VAR_TYPE: support_data UNITS: "MeV" - VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMAX: 20.0 + VALIDMIN: 0.0 DEPEND_1: codice_hi_energy_center codice_hi_energy_plus_delta: @@ -186,38 +184,10 @@ codice_hi_energy_plus_delta: FIELDNAM: CoDICE-Hi proton energy bin upper delta VAR_TYPE: support_data UNITS: "MeV" - VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMAX: 20.0 + VALIDMIN: 0.0 DEPEND_1: codice_hi_energy_center -codice_hi_energy_center: - <<: *default_float32 - CATDESC: CoDICE-Hi proton energy bin (MeV) - FIELDNAM: CoDICE-Hi proton energy bin - VAR_TYPE: support_data - UNITS: "MeV" - DELTA_PLUS_VAR: codice_hi_energy_plus_delta - DELTA_MINUS_VAR: codice_hi_energy_minus_delta - -codice_hi_spin_sector: - CATDESC: CoDICE-Hi spin sector index - FIELDNAM: CoDICE-Hi spin sector index - FILLVAL: -1 - FORMAT: I2 - LABLAXIS: " " - SCALETYP: linear - UNITS: " " - VALIDMIN: 0 - VALIDMAX: 3 - VAR_TYPE: support_data - -swe_electron_energy: - <<: *default_float32 - CATDESC: SWE electron channels (nominal central energies) - FIELDNAM: SWE electron channels - VAR_TYPE: support_data - UNITS: "eV" - codice_hi_elevation: <<: *default_float32 CATDESC: Elevation Angle @@ -229,19 +199,6 @@ codice_hi_elevation: VALIDMAX: 180.0 VAR_TYPE: support_data -codice_hi_elevation_labels: - CATDESC: Elevation Angle - FIELDNAM: Elevation Angle - FORMAT: A6 - VAR_TYPE: metadata - -codice_hi_spin_sector_labels: - CATDESC: Spin Sector - DEPEND_1: spin_sector - FIELDNAM: Spin Sector - FORMAT: A2 - VAR_TYPE: metadata - codice_hi_spin_angle: <<: *default_float32 VAR_TYPE: support_data @@ -251,12 +208,33 @@ codice_hi_spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 - DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle DEPEND_1: codice_hi_spin_sector DEPEND_2: codice_hi_elevation LABL_PTR_1: codice_hi_spin_sector_labels LABL_PTR_2: codice_hi_elevation_labels +codice_hi_spin_sector: + CATDESC: CoDICE-Hi spin sector index + FIELDNAM: CoDICE-Hi spin sector index + FORMAT: I2 + LABLAXIS: " " + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 3 + VAR_TYPE: support_data + +swe_electron_energy: + <<: *default_float32 + CATDESC: SWE electron channels (nominal central energies) + FIELDNAM: SWE electron channels + LABLAXIS: SWE electron channels + VAR_TYPE: support_data + UNITS: "eV" + VALIDMIN: 0.0 + VALIDMAX: 2000.0 + +# Variables codice_hi_h: <<: *default_float32 CATDESC: Proton intensity in 4 spin angles, 15 energy ranges, 4 spin sectors, and 4 elevations @@ -459,42 +437,6 @@ hit_he_omni_high_en: VALIDMAX: 1000000000 VAR_NOTES: Omni indicates the sum of the number of particles divided by the area of the full sky. -mag_B_GSE: - <<: *default_float32 - CATDESC: Magnetic field vector in GSE coordinates - FIELDNAM: B vector in GSE coordinates - LABLAXIS: B (GSE) - DEPEND_0: mag_epoch - LABL_PTR_1: B_GSE_labels - UNITS: nT - FORMAT: F13.6 - VALIDMIN: -65000.0 - VALIDMAX: 65000.0 - -mag_B_GSM: - <<: *default_float32 - CATDESC: Magnetic field vector in GSM coordinates - FIELDNAM: B vector in GSM coordinates - LABLAXIS: B (GSM) - DEPEND_0: mag_epoch - LABL_PTR_1: B_GSM_labels - UNITS: nT - FORMAT: F13.6 - VALIDMIN: -65000.0 - VALIDMAX: 65000.0 - -mag_B_RTN: - <<: *default_float32 - CATDESC: Magnetic field vector in RTN coordinates - FIELDNAM: B vector in RTN coordinates - LABLAXIS: B (RTN) - DEPEND_0: mag_epoch - LABL_PTR_1: B_RTN_labels - UNITS: nT - FORMAT: F13.6 - VALIDMIN: -65000.0 - VALIDMAX: 65000.0 - mag_B_magnitude: <<: *default_float32 CATDESC: Magnitude of the magnetic field vector @@ -505,21 +447,11 @@ mag_B_magnitude: VALIDMIN: 0.0 VALIDMAX: 65000.0 -mag_phi_B_GSM: - <<: *default_float32 - CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates - FIELDNAM: B azimuth angle in GSM coordinates - LABLAXIS: B azimuth (GSM) - DEPEND_0: mag_epoch - UNITS: degrees - VALIDMIN: 0.0 - VALIDMAX: 360.0 - -mag_theta_B_GSM: +mag_theta_B_GSE: <<: *default_float32 - CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates - FIELDNAM: B elevation angle in GSM coordinates - LABLAXIS: B elevation (GSM) + CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates + FIELDNAM: B elevation angle in GSE coordinates + LABLAXIS: B elevation (GSE) DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: -90.0 @@ -535,55 +467,61 @@ mag_phi_B_GSE: VALIDMIN: 0.0 VALIDMAX: 360.0 -mag_theta_B_GSE: +mag_theta_B_GSM: <<: *default_float32 - CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates - FIELDNAM: B elevation angle in GSE coordinates - LABLAXIS: B elevation (GSE) + CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates + FIELDNAM: B elevation angle in GSM coordinates + LABLAXIS: B elevation (GSM) DEPEND_0: mag_epoch UNITS: degrees VALIDMIN: -90.0 VALIDMAX: 90.0 -sc_position_GSM: +mag_phi_B_GSM: <<: *default_float32 - CATDESC: Spacecraft position vector in GSM coordinates - FIELDNAM: Spacecraft position in GSM coordinates - LABLAXIS: SC position (GSM) - UNITS: km - LABL_PTR_1: sc_GSM_labels - VALIDMIN: -3000000.0 - VALIDMAX: 3000000.0 + CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates + FIELDNAM: B azimuth angle in GSM coordinates + LABLAXIS: B azimuth (GSM) + DEPEND_0: mag_epoch + UNITS: degrees + VALIDMIN: 0.0 + VALIDMAX: 360.0 -sc_velocity_GSM: +mag_B_GSE: <<: *default_float32 - CATDESC: Spacecraft velocity vector in GSM coordinates - FIELDNAM: Spacecraft velocity in GSM - LABLAXIS: SC velocity (GSM) - UNITS: km/s - LABL_PTR_1: sc_GSM_labels - VALIDMIN: -1000.0 - VALIDMAX: 1000.0 + CATDESC: Magnetic field vector in GSE coordinates + FIELDNAM: B vector in GSE coordinates + LABLAXIS: B (GSE) + DEPEND_0: mag_epoch + LABL_PTR_1: B_GSE_labels + UNITS: nT + FORMAT: F13.6 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 -sc_position_GSE: +mag_B_GSM: <<: *default_float32 - CATDESC: Spacecraft position vector in GSE coordinates - FIELDNAM: Spacecraft position in GSE - LABLAXIS: SC position (GSE) - UNITS: km - LABL_PTR_1: sc_GSE_labels - VALIDMIN: -3000000.0 - VALIDMAX: 3000000.0 + CATDESC: Magnetic field vector in GSM coordinates + FIELDNAM: B vector in GSM coordinates + LABLAXIS: B (GSM) + DEPEND_0: mag_epoch + LABL_PTR_1: B_GSM_labels + UNITS: nT + FORMAT: F13.6 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 -sc_velocity_GSE: +mag_B_RTN: <<: *default_float32 - CATDESC: Spacecraft velocity vector in GSE coordinates - FIELDNAM: Spacecraft velocity in GSE - LABLAXIS: SC velocity (GSE) - UNITS: km/s - LABL_PTR_1: sc_GSE_labels - VALIDMIN: -1000.0 - VALIDMAX: 1000.0 + CATDESC: Magnetic field vector in RTN coordinates + FIELDNAM: B vector in RTN coordinates + LABLAXIS: B (RTN) + DEPEND_0: mag_epoch + LABL_PTR_1: B_RTN_labels + UNITS: nT + FORMAT: F13.6 + VALIDMIN: -65000.0 + VALIDMAX: 65000.0 swapi_pseudo_proton_density: <<: *default_float32 @@ -623,7 +561,7 @@ swe_normalized_counts: UNITS: Normalized counts VALIDMIN: 0 VALIDMAX: 500000 - DEPEND_1: swe_electron_energy_labels + DEPEND_1: swe_electron_energy DISPLAY_TYPE: stack_plot LABLAXIS: Electrons VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. @@ -637,3 +575,43 @@ swe_counterstreaming_electrons: VALIDMIN: 0 VALIDMAX: 1 VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow. + +sc_position_GSE: + <<: *default_float32 + CATDESC: Spacecraft position vector in GSE coordinates + FIELDNAM: Spacecraft position in GSE + LABLAXIS: SC position (GSE) + UNITS: km + LABL_PTR_1: sc_GSE_labels + VALIDMIN: -3000000.0 + VALIDMAX: 3000000.0 + +sc_velocity_GSE: + <<: *default_float32 + CATDESC: Spacecraft velocity vector in GSE coordinates + FIELDNAM: Spacecraft velocity in GSE + LABLAXIS: SC velocity (GSE) + UNITS: km/s + LABL_PTR_1: sc_GSE_labels + VALIDMIN: -1000.0 + VALIDMAX: 1000.0 + +sc_position_GSM: + <<: *default_float32 + CATDESC: Spacecraft position vector in GSM coordinates + FIELDNAM: Spacecraft position in GSM coordinates + LABLAXIS: SC position (GSM) + UNITS: km + LABL_PTR_1: sc_GSM_labels + VALIDMIN: -3000000.0 + VALIDMAX: 3000000.0 + +sc_velocity_GSM: + <<: *default_float32 + CATDESC: Spacecraft velocity vector in GSM coordinates + FIELDNAM: Spacecraft velocity in GSM + LABLAXIS: SC velocity (GSM) + UNITS: km/s + LABL_PTR_1: sc_GSM_labels + VALIDMIN: -1000.0 + VALIDMAX: 1000.0 diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index e60fe9de43..8f8b514068 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -6,7 +6,7 @@ # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins "codice_hi_h": [ "codice_hi_epoch", - "codice_hi_h_energy_range_labels", + "codice_hi_energy_center", "codice_hi_spin_sector", "codice_hi_elevation", ], @@ -65,13 +65,13 @@ # SWE Counterstreaming flag "swe_counterstreaming_electrons": ["swe_epoch"], # Spacecraft position in GSE coordinates - "sc_position_GSE": ["spacecraft_epoch", "sc_GSE_labels"], + "sc_position_GSE": ["epoch", "sc_GSE_labels"], # Spacecraft velocity in GSE coordinates - "sc_velocity_GSE": ["spacecraft_epoch", "sc_GSE_labels"], + "sc_velocity_GSE": ["epoch", "sc_GSE_labels"], # Spacecraft position in GSM coordinates - "sc_position_GSM": ["spacecraft_epoch", "sc_GSM_labels"], + "sc_position_GSM": ["epoch", "sc_GSM_labels"], # Spacecraft velocity in GSM coordinates - "sc_velocity_GSM": ["spacecraft_epoch", "sc_GSM_labels"], + "sc_velocity_GSM": ["epoch", "sc_GSM_labels"], } IALIRT_DTYPES = { @@ -117,7 +117,14 @@ "sc_velocity_GSE": np.float32, } -codice_hi_energy_centers = [ +hit_restricted_fields = { + "hit_e_a_side_high_en", + "hit_e_b_side_high_en", + "hit_h_a_side_high_en", + "hit_h_b_side_high_en", +} + +codice_hi_energy_center = [ 0.02378414, 0.03363586, 0.04756828, diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index a149e87821..9c57dd5c3e 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -13,9 +13,10 @@ from imap_processing.ialirt.utils.constants import ( IALIRT_DIMS, IALIRT_DTYPES, - codice_hi_energy_centers, + codice_hi_energy_center, codice_hi_energy_minus, codice_hi_energy_plus, + hit_restricted_fields, swe_energy, ) @@ -47,7 +48,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for record in records: inst = record.get("instrument") by_inst[record["instrument"]].append(record) - if inst in one_epoch: + if inst == "spacecraft": + epochs[inst].append(record["ttj2000ns"]) + elif inst in one_epoch: epochs[inst].append(record[f"{inst}_epoch"]) elif inst in multi_epoch: epochs[inst].extend(record[f"{inst}_epoch"]) @@ -55,14 +58,16 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 epoch_arrays = {} for inst, arr in epochs.items(): - coord = f"{inst}_epoch" - epoch_arrays[coord] = xr.DataArray( - data=arr, - name=coord, - dims=[coord], - attrs=cdf_manager.get_variable_attributes( + if inst == "spacecraft": + coord = "epoch" + attr = cdf_manager.get_variable_attributes("epoch", check_schema=False) + else: + coord = f"{inst}_epoch" + attr = cdf_manager.get_variable_attributes( f"{inst}_epoch", check_schema=False - ), + ) + epoch_arrays[coord] = xr.DataArray( + data=arr, name=coord, dims=[coord], attrs=attr ) sc_gsm_component = xr.DataArray( @@ -101,7 +106,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) swe_electron_energy = xr.DataArray( - data=swe_energy, + data=np.float32(swe_energy), name="swe_electron_energy", dims=["swe_electron_energy"], attrs=cdf_manager.get_variable_attributes( @@ -109,8 +114,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - codice_hi_energy_center = xr.DataArray( - data=codice_hi_energy_centers, + codice_hi_energy_centers = xr.DataArray( + data=np.float32(codice_hi_energy_center), name="codice_hi_energy_center", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -119,7 +124,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) codice_energy_minus = xr.DataArray( - data=codice_hi_energy_minus, + data=np.float32(codice_hi_energy_minus), name="codice_hi_energy_minus", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -128,7 +133,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) codice_energy_plus = xr.DataArray( - data=codice_hi_energy_plus, + data=np.float32(codice_hi_energy_plus), name="codice_hi_energy_plus", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -137,7 +142,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) elevation = xr.DataArray( - HI_IALIRT_ELEVATION_ANGLE, + np.float32(HI_IALIRT_ELEVATION_ANGLE), name="codice_hi_elevation", dims=["codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( @@ -155,7 +160,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 HI_IALIRT_REF_SPIN_ANGLE[:, np.newaxis] + np.array([0, 1, 2, 3]) * 90 ) % 360.0 spin_angle = xr.DataArray( - data=spin_angles, + data=np.float32(spin_angles), name="codice_hi_spin_angle", dims=["codice_hi_spin_sector", "codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( @@ -177,21 +182,21 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "codice_lo_epoch": epoch_arrays["codice_lo_epoch"], "hit_epoch": epoch_arrays["hit_epoch"], "mag_epoch": epoch_arrays["mag_epoch"], - "spacecraft_epoch": epoch_arrays["spacecraft_epoch"], "swapi_epoch": epoch_arrays["swapi_epoch"], "swe_epoch": epoch_arrays["swe_epoch"], + "epoch": epoch_arrays["epoch"], "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, "sc_GSM_labels": sc_gsm_component, "sc_GSE_labels": sc_gse_component, - "swe_electron_energy": swe_electron_energy, - "codice_hi_energy_center": codice_hi_energy_center, + "codice_hi_energy_center": codice_hi_energy_centers, "codice_hi_energy_minus_delta": codice_energy_minus, "codice_hi_energy_plus_delta": codice_energy_plus, "codice_hi_elevation": elevation, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, + "swe_electron_energy": swe_electron_energy, } dataset = xr.Dataset( coords=coords, @@ -238,7 +243,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for i, record in enumerate(by_inst.get("hit", [])): for key in IALIRT_DIMS.keys(): - if key.startswith("hit_"): + if key.startswith("hit_") and key not in hit_restricted_fields: dataset[key].data[i] = np.uint32(record[key]) for i, record in enumerate(by_inst.get("swapi", [])): diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 85b7fef9b9..e0001fb588 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -6,6 +6,7 @@ import numpy.testing as npt from imap_processing.cdf.utils import write_cdf +from imap_processing.ialirt.utils.constants import swe_energy from imap_processing.ialirt.utils.create_xarray import create_xarray_from_records @@ -13,137 +14,152 @@ def test_create_dataset(): """Tests create_dataset function.""" records = [ { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000000, - "instrument": "codice_hi", - "codice_hi_h": [ - [ - [[Decimal("0.0") for _ in range(4)] for _ in range(4)] - for _ in range(15) - ] - for _ in range(4) - ], - "codice_hi_epoch": [ - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - ], + "instrument": "mag", + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000001, + "mag_epoch": 123456789000001, + "mag_B_GSE": [Decimal("5.0"), Decimal("-3.2"), Decimal("1.1")], + "mag_B_GSM": [Decimal("4.8"), Decimal("-3.0"), Decimal("1.0")], + "mag_B_RTN": [Decimal("5.1"), Decimal("-3.3"), Decimal("1.2")], + "mag_B_magnitude": Decimal("6.0"), + "mag_phi_B_GSM": Decimal("45.0"), + "mag_theta_B_GSM": Decimal("30.0"), + "mag_phi_B_GSE": Decimal("50.0"), + "mag_theta_B_GSE": Decimal("35.0"), }, { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000000, "instrument": "codice_hi", + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000000, + "codice_hi_epoch": [ + 123456789000000, + 123456789000000, + 123456789000000, + 123456789000000, + ], "codice_hi_h": [ [ - [[Decimal("0.0") for _ in range(4)] for _ in range(4)] + [[Decimal("1.0") for _ in range(4)] for _ in range(4)] for _ in range(15) ] for _ in range(4) ], - "codice_hi_epoch": [ - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - Decimal("0.0"), - ], }, { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000000, "instrument": "codice_lo", - "codice_hi_epoch": Decimal("0.0"), - "codice_lo_c_over_o_abundance": Decimal("0.0"), - "codice_lo_mg_over_o_abundance": Decimal("0.0"), - "codice_lo_fe_over_o_abundance": Decimal("0.0"), - "codice_lo_c_plus_6_over_c_plus_5": Decimal("0.0"), - "codice_lo_o_plus_7_over_o_plus_6": Decimal("0.0"), - "codice_lo_fe_low_over_fe_high": Decimal("0.0"), + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000000, + "codice_lo_epoch": 123456789000000, + "codice_lo_c_over_o_abundance": Decimal("0.5"), + "codice_lo_mg_over_o_abundance": Decimal("0.3"), + "codice_lo_fe_over_o_abundance": Decimal("0.2"), + "codice_lo_c_plus_6_over_c_plus_5": Decimal("0.7"), + "codice_lo_o_plus_7_over_o_plus_6": Decimal("0.6"), + "codice_lo_fe_low_over_fe_high": Decimal("0.4"), }, { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000001, - "instrument": "mag", - "mag_epoch": Decimal("0.0"), - "mag_B_GSE": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_GSM": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_RTN": [Decimal("0.0"), Decimal("0.0"), Decimal("0.0")], - "mag_B_magnitude": Decimal("0.0"), - "mag_phi_B_GSM": Decimal("0.0"), - "mag_theta_B_GSM": Decimal("0.0"), - "mag_phi_B_GSE": Decimal("0.0"), - "mag_theta_B_GSE": Decimal("0.0"), + "instrument": "hit", + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000002, + "hit_epoch": 123456789000002, + "hit_e_a_side_low_en": Decimal("0.0"), + "hit_e_a_side_med_en": Decimal("0.0"), + "hit_e_a_side_high_en": Decimal("0.0"), + "hit_e_b_side_low_en": Decimal("0.0"), + "hit_e_b_side_med_en": Decimal("0.0"), + "hit_e_b_side_high_en": Decimal("0.0"), + "hit_h_omni_low_en": Decimal("0.0"), + "hit_h_omni_med_en": Decimal("0.0"), + "hit_h_a_side_high_en": Decimal("0.0"), + "hit_h_b_side_high_en": Decimal("0.0"), + "hit_he_omni_low_en": Decimal("0.0"), + "hit_he_omni_high_en": Decimal("0.0"), }, { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000002, "instrument": "swapi", - "swapi_pseudo_proton_speed": Decimal("0.0"), - "swapi_pseudo_proton_density": Decimal("0.0"), - "swapi_pseudo_proton_temperature": Decimal("0.0"), + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000002, + "swapi_epoch": 123456789000002, + "swapi_pseudo_proton_speed": Decimal("400.0"), + "swapi_pseudo_proton_density": Decimal("5.0"), + "swapi_pseudo_proton_temperature": Decimal("100000.0"), }, { - "apid": 478, - "met": 123456789, - "met_in_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000002, "instrument": "swe", + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000002, + "swe_epoch": 123456789000002, "swe_normalized_counts": [Decimal("0.0") for _ in range(8)], - "swe_counterstreaming_electrons": Decimal("0.0"), + "swe_counterstreaming_electrons": Decimal("1.0"), + }, + { + "instrument": "spacecraft", + "time_utc": "2025-10-29T18:55:02", + "ttj2000ns": 123456789000002, + "sc_position_GSE": [ + Decimal("1373251.6968303905"), + Decimal("-431299.0150430931"), + Decimal("73446.43257187483"), + ], + "sc_position_GSM": [ + Decimal("1373251.6968303905"), + Decimal("-400988.5784292875"), + Decimal("174989.6534196707"), + ], + "sc_velocity_GSE": [ + Decimal("0.03919581036966908"), + Decimal("-0.21796820670587755"), + Decimal("-0.019698638532273577"), + ], + "sc_velocity_GSM": [ + Decimal("0.03919581036966908"), + Decimal("-1.9156200243319468"), + Decimal("-3.8606800975317896"), + ], }, ] dataset = create_xarray_from_records(records) - assert (dataset["component"].values == ["x", "y", "z"]).all() - assert (dataset["RTN_component"].values == ["radial", "tangential", "normal"]).all() - npt.assert_array_equal(dataset["esa_step"].values, np.arange(8)) + assert ( + dataset["B_GSM_labels"].values == ["Bx (GSM)", "By (GSM)", "Bz (GSM)"] + ).all() + assert ( + dataset["B_RTN_labels"].values + == ["B radial (RTN)", "B tangential (RTN)", "B normal (RTN)"] + ).all() + np.testing.assert_allclose( + dataset["swe_electron_energy"].values, + np.array(swe_energy), + rtol=1e-7, + atol=1e-6, + ) npt.assert_array_equal( dataset["swe_normalized_counts"].values[0], np.zeros(8, dtype=np.uint32), ) - npt.assert_array_equal( - dataset["swe_normalized_counts"].values[1], - np.full(8, 4294967295, dtype=np.uint32), - ) np.testing.assert_allclose( dataset["hit_e_a_side_low_en"].values, - [0, 4294967295, 4294967295], + [0.0], ) np.testing.assert_allclose( - dataset["mag_B_GSE"].isel(epoch=0).values, - [0, 0, 0], + dataset["mag_B_GSE"].sel(mag_epoch=123456789000001).values, + [5.0, -3.2, 1.1], ) - np.testing.assert_allclose( - dataset["mag_B_GSE"].isel(epoch=1).values, - [0, 0, 0], - ) - - expected_zeros = np.zeros((15, 4, 4), dtype=np.float32) - expected_fill = np.full((15, 4, 4), -1e31, dtype=np.float32) - - npt.assert_array_equal(dataset["codice_hi_h"].isel(epoch=0).values, expected_zeros) - npt.assert_array_equal(dataset["codice_hi_h"].isel(epoch=1).values, expected_fill) + expected_zeros = np.ones((15, 4, 4), dtype=np.float32) + npt.assert_array_equal( + dataset["codice_hi_h"].isel(codice_hi_epoch=0).values, expected_zeros + ) - assert dataset["mag_B_GSE"].dims == ("epoch", "component") - assert dataset["swe_normalized_counts"].dims == ("epoch", "esa_step") + assert dataset["mag_B_GSE"].dims == ("mag_epoch", "B_GSE_labels") + assert dataset["swe_normalized_counts"].dims == ("swe_epoch", "swe_electron_energy") assert dataset["codice_hi_h"].dims == ( "codice_hi_epoch", - "codice_hi_h_energy_ranges", - "codice_hi_h_elevation", - "codice_hi_h_spin_angle", + "codice_hi_energy_center", + "codice_hi_spin_sector", + "codice_hi_elevation", ) # Tests that you can write to a cdf. From 269d5f199b1bc6399dbb983241544253ef43495b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 12:41:30 -0700 Subject: [PATCH 25/42] updates based on Andriy --- imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 79d19beacf..3665417cac 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -238,8 +238,8 @@ swe_electron_energy: codice_hi_h: <<: *default_float32 CATDESC: Proton intensity in 4 spin angles, 15 energy ranges, 4 spin sectors, and 4 elevations - FIELDNAM: H intensity - LABLAXIS: H intensity + FIELDNAM: H+ intensity + LABLAXIS: H+ intensity DEPEND_0: codice_hi_epoch UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 From 776658e89fd7f2e61a5def729f68387402160d52 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 13:02:05 -0700 Subject: [PATCH 26/42] center of time variable --- imap_processing/ialirt/l0/process_codice.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 1dcef0961b..b9f01a07cd 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -471,12 +471,15 @@ def process_codice( ) l2_lo = calculate_ratios(l1b_lo, l2_lut_path, l2_geometric_factor_path) + # Calculate the center of the time variable + sc_met = int(np.mean([np.min(met), np.max(met)])) + codice_lo_data.append( { "apid": 478, - "met": int(met[0]), - "met_in_utc": met_to_utc(met[0]).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met[0])), + "met": int(sc_met), + "met_in_utc": met_to_utc(sc_met).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", f"{sensor}_c_over_o_abundance": l2_lo.c_over_o_abundance, f"{sensor}_mg_over_o_abundance": l2_lo.mg_over_o_abundance, From ea38b4e2fa5c9bfc105a37b7f58bb6b3d7c140b2 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 13:56:54 -0700 Subject: [PATCH 27/42] update attr --- .../config/imap_ialirt_l1_variable_attrs.yaml | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 3665417cac..6ed232a02f 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -158,7 +158,6 @@ sc_GSE_labels: # Coordinate Variables codice_hi_energy_center: - <<: *default_float32 CATDESC: CoDICE-Hi proton energy bin (MeV) FIELDNAM: CoDICE-Hi proton energy bin VAR_TYPE: support_data @@ -167,16 +166,21 @@ codice_hi_energy_center: VALIDMIN: 0.0 DELTA_PLUS_VAR: codice_hi_energy_plus_delta DELTA_MINUS_VAR: codice_hi_energy_minus_delta + FILLVAL: -1.0e31 + FORMAT: F12.6 + dtype: float32 codice_hi_energy_minus_delta: - <<: *default_float32 CATDESC: CoDICE-Hi proton energy bin lower delta (MeV) FIELDNAM: CoDICE-Hi proton energy bin lower delta VAR_TYPE: support_data UNITS: "MeV" VALIDMAX: 20.0 VALIDMIN: 0.0 - DEPEND_1: codice_hi_energy_center + DEPEND_0: codice_hi_energy_center + FILLVAL: -1.0e31 + FORMAT: F12.6 + dtype: float32 codice_hi_energy_plus_delta: <<: *default_float32 @@ -186,10 +190,12 @@ codice_hi_energy_plus_delta: UNITS: "MeV" VALIDMAX: 20.0 VALIDMIN: 0.0 - DEPEND_1: codice_hi_energy_center + DEPEND_0: codice_hi_energy_center + FILLVAL: -1.0e31 + FORMAT: F12.6 + dtype: float32 codice_hi_elevation: - <<: *default_float32 CATDESC: Elevation Angle FIELDNAM: Elevation Angle LABLAXIS: Elevation Angle @@ -198,6 +204,20 @@ codice_hi_elevation: VALIDMIN: 0.0 VALIDMAX: 180.0 VAR_TYPE: support_data + FILLVAL: -1.0e31 + FORMAT: F12.6 + dtype: float32 + +codice_hi_spin_sector: + CATDESC: CoDICE-Hi spin sector index + FIELDNAM: CoDICE-Hi spin sector index + FORMAT: I2 + LABLAXIS: " " + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 3 + VAR_TYPE: support_data codice_hi_spin_angle: <<: *default_float32 @@ -208,24 +228,11 @@ codice_hi_spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 + DEPEND_0: codice_hi_epoch DEPEND_1: codice_hi_spin_sector DEPEND_2: codice_hi_elevation - LABL_PTR_1: codice_hi_spin_sector_labels - LABL_PTR_2: codice_hi_elevation_labels - -codice_hi_spin_sector: - CATDESC: CoDICE-Hi spin sector index - FIELDNAM: CoDICE-Hi spin sector index - FORMAT: I2 - LABLAXIS: " " - SCALETYP: linear - UNITS: " " - VALIDMIN: 0 - VALIDMAX: 3 - VAR_TYPE: support_data swe_electron_energy: - <<: *default_float32 CATDESC: SWE electron channels (nominal central energies) FIELDNAM: SWE electron channels LABLAXIS: SWE electron channels @@ -233,6 +240,9 @@ swe_electron_energy: UNITS: "eV" VALIDMIN: 0.0 VALIDMAX: 2000.0 + FILLVAL: -1.0e31 + FORMAT: F12.6 + dtype: float32 # Variables codice_hi_h: @@ -244,9 +254,9 @@ codice_hi_h: UNITS: "counts / cm2-sr-s-MeV" VALIDMIN: 0 VALIDMAX: 100000000.0 - DEPEND_1: codice_hi_h_energy_range - DEPEND_2: codice_hi_h_spin_sector - DEPEND_3: codice_hi_h_elevation + DEPEND_1: codice_hi_energy_center + DEPEND_2: codice_hi_spin_sector + DEPEND_3: codice_hi_elevation DISPLAY_TYPE: no_plot FORMAT: F16.6 VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. From c2f914eab2b5b32519f1cae19905008f10084517 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 14:30:47 -0700 Subject: [PATCH 28/42] update attrs --- .../config/imap_ialirt_l1_variable_attrs.yaml | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 6ed232a02f..fdf1986405 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -259,7 +259,7 @@ codice_hi_h: DEPEND_3: codice_hi_elevation DISPLAY_TYPE: no_plot FORMAT: F16.6 - VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200–0.0283, 0.0283–0.0400, 0.0400–0.0566, 0.0566–0.0800, 0.0800–0.113, 0.113–0.160, 0.160–0.226, 0.226–0.320, 0.320–0.453, 0.453–0.640, 0.640–0.905, 0.905–1.28, 1.28–1.81, 1.81–2.56, 2.56–3.62. + VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200 to 0.0283, 0.0283 to 0.0400, 0.0400 to 0.0566, 0.0566 to 0.0800, 0.0800 to 0.113, 0.113 to 0.160, 0.160 to 0.226, 0.226 to 0.320, 0.320 to 0.453, 0.453 to 0.640, 0.640 to 0.905, 0.905 to 1.28, 1.28 to 1.81, 1.81 to 2.56, 2.56 to 3.62. codice_lo_c_over_o_abundance: <<: *default_float32 @@ -389,9 +389,9 @@ hit_e_b_side_high_en: hit_h_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6–8 MeV) protons (omnidirectional) - FIELDNAM: Protons 6–8 MeV (omni) - LABLAXIS: Omni H 6–8 MeV + CATDESC: Low energy (6 to 8 MeV) protons (omnidirectional) + FIELDNAM: Protons 6 to 8 MeV (omni) + LABLAXIS: Omni H 6 to 8 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -399,9 +399,9 @@ hit_h_omni_low_en: hit_h_omni_med_en: <<: *default_uint32 - CATDESC: Medium energy (12–15 MeV) protons (omnidirectional) - FIELDNAM: Protons 12–15 MeV (omni) - LABLAXIS: Omni H 12–15 MeV + CATDESC: Medium energy (12 to 15 MeV) protons (omnidirectional) + FIELDNAM: Protons 12 to 15 MeV (omni) + LABLAXIS: Omni H 12 to 15 MeV UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -429,9 +429,9 @@ hit_h_b_side_high_en: hit_he_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6–8 MeV/nuc) helium (omnidirectional) - FIELDNAM: Helium 6–8 MeV/nuc (omni) - LABLAXIS: Omni He 6–8 MeV/nuc + CATDESC: Low energy (6 to 8 MeV/nuc) helium (omnidirectional) + FIELDNAM: Helium 6 to 8 MeV/nuc (omni) + LABLAXIS: Omni He 6 to 8 MeV/nuc UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -439,9 +439,9 @@ hit_he_omni_low_en: hit_he_omni_high_en: <<: *default_uint32 - CATDESC: High energy (15–70 MeV/nuc) helium (omnidirectional) - FIELDNAM: Helium 15–70 MeV/nuc (omni) - LABLAXIS: Omni He 15–70 MeV/nuc + CATDESC: High energy (15 to 70 MeV/nuc) helium (omnidirectional) + FIELDNAM: Helium 15 to 70 MeV/nuc (omni) + LABLAXIS: Omni He 15 to 70 MeV/nuc UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -459,7 +459,7 @@ mag_B_magnitude: mag_theta_B_GSE: <<: *default_float32 - CATDESC: Elevation angle (θ) of the magnetic field in GSE coordinates + CATDESC: Elevation angle of the magnetic field in GSE coordinates FIELDNAM: B elevation angle in GSE coordinates LABLAXIS: B elevation (GSE) DEPEND_0: mag_epoch @@ -469,7 +469,7 @@ mag_theta_B_GSE: mag_phi_B_GSE: <<: *default_float32 - CATDESC: Azimuth angle (φ) of the magnetic field in GSE coordinates + CATDESC: Azimuth angle of the magnetic field in GSE coordinates FIELDNAM: B azimuth angle in GSE coordinates LABLAXIS: B azimuth (GSE) DEPEND_0: mag_epoch @@ -479,7 +479,7 @@ mag_phi_B_GSE: mag_theta_B_GSM: <<: *default_float32 - CATDESC: Elevation angle (θ) of the magnetic field in GSM coordinates + CATDESC: Elevation angle of the magnetic field in GSM coordinates FIELDNAM: B elevation angle in GSM coordinates LABLAXIS: B elevation (GSM) DEPEND_0: mag_epoch @@ -489,7 +489,7 @@ mag_theta_B_GSM: mag_phi_B_GSM: <<: *default_float32 - CATDESC: Azimuth angle (φ) of the magnetic field in GSM coordinates + CATDESC: Azimuth angle of the magnetic field in GSM coordinates FIELDNAM: B azimuth angle in GSM coordinates LABLAXIS: B azimuth (GSM) DEPEND_0: mag_epoch @@ -562,6 +562,7 @@ swapi_pseudo_proton_temperature: VALIDMIN: 0.0 VALIDMAX: 50000000.0 VAR_NOTES: The pseudo temperature is derived using a simplified analytical model of solar wind proton speed. + FORMAT: F15.6 # SWE Normalized Counts swe_normalized_counts: From 6643102cdcb60dcea35e048e0ab1c367470620b7 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 14:33:49 -0700 Subject: [PATCH 29/42] update attrs --- imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index fdf1986405..764cc594ba 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -318,7 +318,7 @@ codice_lo_o_plus_7_over_o_plus_6: codice_lo_fe_low_over_fe_high: <<: *default_float32 - CATDESC: Fe low/high charge state ratio (low=6–12, high=13–17) + CATDESC: Fe low/high charge state ratio (low=6 to 12, high=13 to 17) FIELDNAM: Fe low/high charge state ratio LABLAXIS: Fe low/high ratio DEPEND_0: codice_lo_epoch From 89c52ce2174087bdaff6b2a78389c1de4288bb9b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 15:09:13 -0700 Subject: [PATCH 30/42] update formatting --- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 13 +++++++++++++ imap_processing/ialirt/utils/constants.py | 2 +- imap_processing/ialirt/utils/create_xarray.py | 15 +++++++++++++-- .../tests/ialirt/unit/test_create_xarray.py | 2 +- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 764cc594ba..9a719c24fa 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -84,6 +84,7 @@ instrument_epoch: &instrument_epoch REFERENCE_POSITION: Rotating Earth Geoid RESOLUTION: ' ' DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" + dtype: int64 codice_hi_epoch: <<: *instrument_epoch @@ -128,12 +129,23 @@ B_GSM_labels: VAR_TYPE: support_data UNITS: " " +B_GSE_component_index: + CATDESC: B GSE component index + FIELDNAM: B GSE component index + FORMAT: I2 + VAR_TYPE: support_data + UNITS: " " + VALIDMIN: 1 + VALIDMAX: 3 + dtype: int8 + B_GSE_labels: CATDESC: B GSE component labels FIELDNAM: B GSE component labels FORMAT: A8 VAR_TYPE: support_data UNITS: " " + DEPEND_0: B_GSE_component_index B_RTN_labels: CATDESC: B RTN component labels B radial (RTN), B tangential (RTN), B normal (RTN) @@ -503,6 +515,7 @@ mag_B_GSE: FIELDNAM: B vector in GSE coordinates LABLAXIS: B (GSE) DEPEND_0: mag_epoch + DEPEND_1: B_GSE_component_index LABL_PTR_1: B_GSE_labels UNITS: nT FORMAT: F13.6 diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 8f8b514068..6d497fdf7b 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -49,7 +49,7 @@ # Azimuth angle (φ) of the magnetic field in GSM coordinates "mag_phi_B_GSM": ["mag_epoch"], # Magnetic field vector in GSE coordinates - "mag_B_GSE": ["mag_epoch", "B_GSE_labels"], + "mag_B_GSE": ["mag_epoch", "B_GSE_component_index"], # Magnetic field vector in GSM coordinates "mag_B_GSM": ["mag_epoch", "B_GSM_labels"], # Magnetic field vector in RTN coordinates diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 9c57dd5c3e..1c59a644c0 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -67,7 +67,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 f"{inst}_epoch", check_schema=False ) epoch_arrays[coord] = xr.DataArray( - data=arr, name=coord, dims=[coord], attrs=attr + data=np.array(arr, dtype=np.int64), name=coord, dims=[coord], attrs=attr ) sc_gsm_component = xr.DataArray( @@ -91,10 +91,20 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("B_GSM_labels", check_schema=False), ) + # numeric component index for GSE vector (1-based; use 0-based if you prefer) + gse_component_index = xr.DataArray( + data=np.arange(1, 4, dtype=np.int32), + name="B_GSE_component_index", + dims=["B_GSE_component_index"], + attrs=cdf_manager.get_variable_attributes( + "B_GSE_component_index", check_schema=False + ), + ) + gse_component = xr.DataArray( ["Bx (GSE)", "By (GSE)", "Bz (GSE)"], name="B_GSE_labels", - dims=["B_GSE_labels"], + dims=["B_GSE_component_index"], attrs=cdf_manager.get_variable_attributes("B_GSE_labels", check_schema=False), ) @@ -186,6 +196,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "swe_epoch": epoch_arrays["swe_epoch"], "epoch": epoch_arrays["epoch"], "B_GSM_labels": gsm_component, + "B_GSE_component_index": gse_component_index, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, "sc_GSM_labels": sc_gsm_component, diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index e0001fb588..396e98fd13 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -153,7 +153,7 @@ def test_create_dataset(): dataset["codice_hi_h"].isel(codice_hi_epoch=0).values, expected_zeros ) - assert dataset["mag_B_GSE"].dims == ("mag_epoch", "B_GSE_labels") + assert dataset["mag_B_GSE"].dims == ("mag_epoch", "B_GSE_component_index") assert dataset["swe_normalized_counts"].dims == ("swe_epoch", "swe_electron_energy") assert dataset["codice_hi_h"].dims == ( "codice_hi_epoch", From 7779bebc598bde8670fbf38518e7ac7d0d9b371f Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 16:09:32 -0700 Subject: [PATCH 31/42] skt editor updates --- .../config/imap_ialirt_l1_variable_attrs.yaml | 55 ++++++++++++++++++- imap_processing/ialirt/utils/constants.py | 12 ++-- imap_processing/ialirt/utils/create_xarray.py | 49 ++++++++++++++++- 3 files changed, 104 insertions(+), 12 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 9a719c24fa..f639022eac 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -122,12 +122,24 @@ epoch: FIELDNAM: Spacecraft epoch # Labels +B_GSM_component_index: + CATDESC: B GSM component index + FIELDNAM: B GSM component index + FORMAT: I2 + VAR_TYPE: support_data + UNITS: " " + VALIDMIN: 1 + VALIDMAX: 3 + dtype: int8 + LABL_PTR_1: B_GSM_labels + B_GSM_labels: CATDESC: B GSM component labels FIELDNAM: B GSM component labels FORMAT: A8 - VAR_TYPE: support_data + VAR_TYPE: metadata UNITS: " " + DEPEND_0: B_GSM_component_index B_GSE_component_index: CATDESC: B GSE component index @@ -138,34 +150,69 @@ B_GSE_component_index: VALIDMIN: 1 VALIDMAX: 3 dtype: int8 + LABL_PTR_1: B_GSE_labels B_GSE_labels: CATDESC: B GSE component labels FIELDNAM: B GSE component labels FORMAT: A8 - VAR_TYPE: support_data + VAR_TYPE: metadata UNITS: " " DEPEND_0: B_GSE_component_index +B_RTN_component_index: + CATDESC: B RTN component index + FIELDNAM: B RTN component index + FORMAT: I2 + VAR_TYPE: support_data + UNITS: " " + VALIDMIN: 1 + VALIDMAX: 3 + dtype: int8 + LABL_PTR_1: B_RTN_labels + B_RTN_labels: CATDESC: B RTN component labels B radial (RTN), B tangential (RTN), B normal (RTN) FIELDNAM: B RTN component labels FORMAT: A18 + VAR_TYPE: metadata + UNITS: " " + DEPEND_0: B_RTN_component_index + +sc_GSM_component_index: + CATDESC: Spacecraft GSM component index + FIELDNAM: Spacecraft GSM component index + FORMAT: I2 VAR_TYPE: support_data UNITS: " " + VALIDMIN: 1 + VALIDMAX: 3 + dtype: int8 + LABL_PTR_1: sc_GSM_labels sc_GSM_labels: CATDESC: Spacecraft position and velocity component labels in GSM coordinates FIELDNAM: Spacecraft GSM component labels FORMAT: A8 + VAR_TYPE: metadata + UNITS: " " + +sc_GSE_component_index: + CATDESC: Spacecraft GSE component index + FIELDNAM: Spacecraft GSE component index + FORMAT: I2 VAR_TYPE: support_data UNITS: " " + VALIDMIN: 1 + VALIDMAX: 3 + dtype: int8 + LABL_PTR_1: sc_GSE_labels sc_GSE_labels: CATDESC: Spacecraft position and velocity component labels in GSE coordinates FIELDNAM: Spacecraft GSE component labels FORMAT: A8 - VAR_TYPE: support_data + VAR_TYPE: metadata UNITS: " " # Coordinate Variables @@ -609,6 +656,7 @@ sc_position_GSE: LABL_PTR_1: sc_GSE_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 + FORMAT: F15.6 sc_velocity_GSE: <<: *default_float32 @@ -629,6 +677,7 @@ sc_position_GSM: LABL_PTR_1: sc_GSM_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 + FORMAT: F15.6 sc_velocity_GSM: <<: *default_float32 diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 6d497fdf7b..2b3bd16309 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -51,9 +51,9 @@ # Magnetic field vector in GSE coordinates "mag_B_GSE": ["mag_epoch", "B_GSE_component_index"], # Magnetic field vector in GSM coordinates - "mag_B_GSM": ["mag_epoch", "B_GSM_labels"], + "mag_B_GSM": ["mag_epoch", "B_GSM_component_index"], # Magnetic field vector in RTN coordinates - "mag_B_RTN": ["mag_epoch", "B_RTN_labels"], + "mag_B_RTN": ["mag_epoch", "B_RTN_component_index"], # Pseudo density of solar wind protons "swapi_pseudo_proton_density": ["swapi_epoch"], # Pseudo speed of solar wind protons in solar inertial frame @@ -65,13 +65,13 @@ # SWE Counterstreaming flag "swe_counterstreaming_electrons": ["swe_epoch"], # Spacecraft position in GSE coordinates - "sc_position_GSE": ["epoch", "sc_GSE_labels"], + "sc_position_GSE": ["epoch", "sc_GSE_component_index"], # Spacecraft velocity in GSE coordinates - "sc_velocity_GSE": ["epoch", "sc_GSE_labels"], + "sc_velocity_GSE": ["epoch", "sc_GSE_component_index"], # Spacecraft position in GSM coordinates - "sc_position_GSM": ["epoch", "sc_GSM_labels"], + "sc_position_GSM": ["epoch", "sc_GSM_component_index"], # Spacecraft velocity in GSM coordinates - "sc_velocity_GSM": ["epoch", "sc_GSM_labels"], + "sc_velocity_GSM": ["epoch", "sc_GSM_component_index"], } IALIRT_DTYPES = { diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 1c59a644c0..45a31c36bc 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -69,6 +69,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 epoch_arrays[coord] = xr.DataArray( data=np.array(arr, dtype=np.int64), name=coord, dims=[coord], attrs=attr ) + # numeric component index for GSE vector (1-based; use 0-based if you prefer) + sc_gsm_component_index = xr.DataArray( + data=np.arange(1, 4, dtype=np.int32), + name="sc_GSM_component_index", + dims=["sc_GSM_component_index"], + attrs=cdf_manager.get_variable_attributes( + "sc_GSM_component_index", check_schema=False + ), + ) sc_gsm_component = xr.DataArray( ["x (GSM)", "y (GSM)", "z (GSM)"], @@ -77,17 +86,37 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("sc_GSM_labels", check_schema=False), ) + # numeric component index for GSE vector (1-based; use 0-based if you prefer) + sc_gse_component_index = xr.DataArray( + data=np.arange(1, 4, dtype=np.int32), + name="sc_GSE_component_index", + dims=["sc_GSE_component_index"], + attrs=cdf_manager.get_variable_attributes( + "sc_GSE_component_index", check_schema=False + ), + ) + sc_gse_component = xr.DataArray( ["x (GSE)", "y (GSE)", "z (GSE)"], name="sc_GSE_labels", - dims=["sc_GSE_labels"], + dims=["sc_GSE_component_index"], attrs=cdf_manager.get_variable_attributes("sc_GSE_labels", check_schema=False), ) + # numeric component index for GSE vector (1-based; use 0-based if you prefer) + gsm_component_index = xr.DataArray( + data=np.arange(1, 4, dtype=np.int32), + name="B_GSM_component_index", + dims=["B_GSM_component_index"], + attrs=cdf_manager.get_variable_attributes( + "B_GSM_component_index", check_schema=False + ), + ) + gsm_component = xr.DataArray( ["Bx (GSM)", "By (GSM)", "Bz (GSM)"], name="B_GSM_labels", - dims=["B_GSM_labels"], + dims=["B_GSM_component_index"], attrs=cdf_manager.get_variable_attributes("B_GSM_labels", check_schema=False), ) @@ -108,10 +137,20 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_variable_attributes("B_GSE_labels", check_schema=False), ) + # numeric component index for GSE vector (1-based; use 0-based if you prefer) + rtn_component_index = xr.DataArray( + data=np.arange(1, 4, dtype=np.int32), + name="B_RTN_component_index", + dims=["B_RTN_component_index"], + attrs=cdf_manager.get_variable_attributes( + "B_RTN_component_index", check_schema=False + ), + ) + rtn_component = xr.DataArray( ["B radial (RTN)", "B tangential (RTN)", "B normal (RTN)"], name="B_RTN_labels", - dims=["B_RTN_labels"], + dims=["B_RTN_component_index"], attrs=cdf_manager.get_variable_attributes("B_RTN_labels", check_schema=False), ) @@ -195,11 +234,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "swapi_epoch": epoch_arrays["swapi_epoch"], "swe_epoch": epoch_arrays["swe_epoch"], "epoch": epoch_arrays["epoch"], + "B_GSM_component_index": gsm_component_index, "B_GSM_labels": gsm_component, "B_GSE_component_index": gse_component_index, "B_GSE_labels": gse_component, + "B_RTN_component_index": rtn_component_index, "B_RTN_labels": rtn_component, + "sc_GSM_component_index": sc_gsm_component_index, "sc_GSM_labels": sc_gsm_component, + "sc_GSE_component_index": sc_gse_component_index, "sc_GSE_labels": sc_gse_component, "codice_hi_energy_center": codice_hi_energy_centers, "codice_hi_energy_minus_delta": codice_energy_minus, From def0b07656a61ce5e84db8b50570ba7a8511d7c7 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 16:28:18 -0700 Subject: [PATCH 32/42] skt editor updates --- .../config/imap_ialirt_l1_variable_attrs.yaml | 21 +++++++++++++------ imap_processing/ialirt/utils/create_xarray.py | 20 +++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index f639022eac..09df0dbf61 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -215,7 +215,6 @@ sc_GSE_labels: VAR_TYPE: metadata UNITS: " " -# Coordinate Variables codice_hi_energy_center: CATDESC: CoDICE-Hi proton energy bin (MeV) FIELDNAM: CoDICE-Hi proton energy bin @@ -223,13 +222,14 @@ codice_hi_energy_center: UNITS: "MeV" VALIDMAX: 20.0 VALIDMIN: 0.0 - DELTA_PLUS_VAR: codice_hi_energy_plus_delta - DELTA_MINUS_VAR: codice_hi_energy_minus_delta + DELTA_PLUS_VAR: codice_hi_energy_plus + DELTA_MINUS_VAR: codice_hi_energy_minus + LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 -codice_hi_energy_minus_delta: +codice_hi_energy_minus: CATDESC: CoDICE-Hi proton energy bin lower delta (MeV) FIELDNAM: CoDICE-Hi proton energy bin lower delta VAR_TYPE: support_data @@ -237,12 +237,12 @@ codice_hi_energy_minus_delta: VALIDMAX: 20.0 VALIDMIN: 0.0 DEPEND_0: codice_hi_energy_center + LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 -codice_hi_energy_plus_delta: - <<: *default_float32 +codice_hi_energy_plus: CATDESC: CoDICE-Hi proton energy bin upper delta (MeV) FIELDNAM: CoDICE-Hi proton energy bin upper delta VAR_TYPE: support_data @@ -250,10 +250,19 @@ codice_hi_energy_plus_delta: VALIDMAX: 20.0 VALIDMIN: 0.0 DEPEND_0: codice_hi_energy_center + LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 +codice_hi_energy_labels: + CATDESC: CoDICE-Hi energy channel labels + FIELDNAM: CoDICE-Hi energy channel labels + FORMAT: A12 + VAR_TYPE: metadata + UNITS: " " + DEPEND_0: codice_hi_energy_center + codice_hi_elevation: CATDESC: Elevation Angle FIELDNAM: Elevation Angle diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 45a31c36bc..19ba0d2c2e 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -164,7 +164,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) codice_hi_energy_centers = xr.DataArray( - data=np.float32(codice_hi_energy_center), + data=np.array(codice_hi_energy_center, dtype=np.float32), name="codice_hi_energy_center", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -173,7 +173,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) codice_energy_minus = xr.DataArray( - data=np.float32(codice_hi_energy_minus), + data=np.array(codice_hi_energy_minus, dtype=np.float32), name="codice_hi_energy_minus", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -182,7 +182,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) codice_energy_plus = xr.DataArray( - data=np.float32(codice_hi_energy_plus), + data=np.array(codice_hi_energy_plus, dtype=np.float32), name="codice_hi_energy_plus", dims=["codice_hi_energy_center"], attrs=cdf_manager.get_variable_attributes( @@ -190,6 +190,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) + codice_energy_labels = xr.DataArray( + [f"ch{n + 1}" for n in range(len(codice_hi_energy_center))], + name="codice_hi_energy_labels", + dims=["codice_hi_energy_center"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_energy_labels", check_schema=False + ), + ) + elevation = xr.DataArray( np.float32(HI_IALIRT_ELEVATION_ANGLE), name="codice_hi_elevation", @@ -245,8 +254,9 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "sc_GSE_component_index": sc_gse_component_index, "sc_GSE_labels": sc_gse_component, "codice_hi_energy_center": codice_hi_energy_centers, - "codice_hi_energy_minus_delta": codice_energy_minus, - "codice_hi_energy_plus_delta": codice_energy_plus, + "codice_hi_energy_minus": codice_energy_minus, + "codice_hi_energy_plus": codice_energy_plus, + "codice_hi_energy_labels": codice_energy_labels, "codice_hi_elevation": elevation, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, From 1d64df3914cbaf5cf54f4203a7580f6117e70217 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 16:36:43 -0700 Subject: [PATCH 33/42] skt editor updates --- .../config/imap_ialirt_l1_variable_attrs.yaml | 20 +++++++++++++------ imap_processing/ialirt/utils/create_xarray.py | 15 ++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 09df0dbf61..ec3d08e582 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -280,12 +280,20 @@ codice_hi_spin_sector: CATDESC: CoDICE-Hi spin sector index FIELDNAM: CoDICE-Hi spin sector index FORMAT: I2 - LABLAXIS: " " - SCALETYP: linear + VAR_TYPE: support_data UNITS: " " VALIDMIN: 0 VALIDMAX: 3 - VAR_TYPE: support_data + dtype: int32 + LABL_PTR_1: codice_hi_spin_sector_labels + +codice_hi_spin_sector_labels: + CATDESC: CoDICE-Hi spin sector labels + FIELDNAM: CoDICE-Hi spin sector labels + FORMAT: A8 + VAR_TYPE: metadata + UNITS: " " + DEPEND_0: codice_hi_spin_sector codice_hi_spin_angle: <<: *default_float32 @@ -296,9 +304,9 @@ codice_hi_spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 - DEPEND_0: codice_hi_epoch - DEPEND_1: codice_hi_spin_sector - DEPEND_2: codice_hi_elevation + DEPEND_0: codice_hi_spin_sector # first dim (numeric index) + DEPEND_1: codice_hi_elevation # second dim (elevation) + LABL_PTR_1: codice_hi_spin_sector_labels swe_electron_energy: CATDESC: SWE electron channels (nominal central energies) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 19ba0d2c2e..bac055a486 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -235,6 +235,20 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) + spin_sector_labels = xr.DataArray( + [ + "sector_0", + "sector_1", + "sector_2", + "sector_3", + ], # or ["sector0","sector1",...] - whatever you prefer + name="codice_hi_spin_sector_labels", + dims=["codice_hi_spin_sector"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_spin_sector_labels", check_schema=False + ), + ) + coords = { "codice_hi_epoch": epoch_arrays["codice_hi_epoch"], "codice_lo_epoch": epoch_arrays["codice_lo_epoch"], @@ -260,6 +274,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "codice_hi_elevation": elevation, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, + "codice_hi_spin_sector_labels": spin_sector_labels, "swe_electron_energy": swe_electron_energy, } dataset = xr.Dataset( From de02f56d725b96f475a1255d457b6b39b356ac52 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 16:43:54 -0700 Subject: [PATCH 34/42] skt editor updates --- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 13 +++++++++++-- imap_processing/ialirt/utils/create_xarray.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index ec3d08e582..d2e97c0e0c 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -276,6 +276,14 @@ codice_hi_elevation: FORMAT: F12.6 dtype: float32 +codice_hi_elevation_labels: + CATDESC: CoDICE-Hi elevation labels + FIELDNAM: CoDICE-Hi elevation labels + FORMAT: A12 + VAR_TYPE: metadata + UNITS: " " + DEPEND_0: codice_hi_elevation + codice_hi_spin_sector: CATDESC: CoDICE-Hi spin sector index FIELDNAM: CoDICE-Hi spin sector index @@ -304,9 +312,10 @@ codice_hi_spin_angle: UNITS: degrees VALIDMAX: 360.0 VALIDMIN: 0.0 - DEPEND_0: codice_hi_spin_sector # first dim (numeric index) - DEPEND_1: codice_hi_elevation # second dim (elevation) + DEPEND_0: codice_hi_spin_sector + DEPEND_1: codice_hi_elevation LABL_PTR_1: codice_hi_spin_sector_labels + LABL_PTR_2: codice_hi_elevation_labels swe_electron_energy: CATDESC: SWE electron channels (nominal central energies) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index bac055a486..e0edd5a87a 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -208,6 +208,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) + elevation_labels = xr.DataArray( + [f"{float(v):.1f}deg" for v in elevation.values], + name="codice_hi_elevation_labels", + dims=["codice_hi_elevation"], + attrs=cdf_manager.get_variable_attributes( + "codice_hi_elevation_labels", check_schema=False + ), + ) + # Calculate spin angle # Formula: # θ_(g,n) = (θ_(g,0)+90°* n) mod 360° @@ -272,6 +281,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "codice_hi_energy_plus": codice_energy_plus, "codice_hi_energy_labels": codice_energy_labels, "codice_hi_elevation": elevation, + "codice_hi_elevation_labels": elevation_labels, "codice_hi_spin_angle": spin_angle, "codice_hi_spin_sector": spin_sector, "codice_hi_spin_sector_labels": spin_sector_labels, From 4b940684e1d741bfa250acc5cc89742837df587b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 17:31:48 -0700 Subject: [PATCH 35/42] skt editor updates --- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 3 ++- imap_processing/ialirt/utils/create_xarray.py | 9 +++++++++ imap_processing/tests/ialirt/unit/test_create_xarray.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index d2e97c0e0c..aee2908ca3 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -72,7 +72,7 @@ instrument_epoch: &instrument_epoch FIELDNAM: epoch LABLAXIS: epoch FILLVAL: -9223372036854775808 - FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + FORMAT: I20 VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end UNITS: ns @@ -85,6 +85,7 @@ instrument_epoch: &instrument_epoch RESOLUTION: ' ' DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" dtype: int64 + DEPEND_0: "" codice_hi_epoch: <<: *instrument_epoch diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index e0edd5a87a..f7fcdeb0e5 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -292,6 +292,15 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"), ) + epoch_keys = [k for k in dataset.coords if k == "epoch" or k.endswith("_epoch")] + + for k in epoch_keys: + # ensure dtype is int64 and set writer hint for TT2000 + data_array = dataset.coords[k].astype(np.int64) + data_array.encoding["dtype"] = np.int64 + data_array.encoding["cdf_type"] = "CDF_TIME_TT2000" + dataset.coords[k] = data_array + # Create variables with fill values. for key in IALIRT_DIMS.keys(): dims = IALIRT_DIMS[key] diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 396e98fd13..8e3da607f2 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -164,6 +164,6 @@ def test_create_dataset(): # Tests that you can write to a cdf. dataset.attrs["Data_version"] = "001" - test_data_path = write_cdf(dataset, istp=True) + test_data_path = write_cdf(dataset, istp=True, compression=None) assert test_data_path.exists() From 24519aa5b3c270c185440e959dfed0be66a4c9ab Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 18:17:15 -0700 Subject: [PATCH 36/42] revert to before time changes --- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 1 - imap_processing/ialirt/utils/create_xarray.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index aee2908ca3..c202df17a5 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -85,7 +85,6 @@ instrument_epoch: &instrument_epoch RESOLUTION: ' ' DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" dtype: int64 - DEPEND_0: "" codice_hi_epoch: <<: *instrument_epoch diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index f7fcdeb0e5..e0edd5a87a 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -292,15 +292,6 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 attrs=cdf_manager.get_global_attributes("imap_ialirt_l1_realtime"), ) - epoch_keys = [k for k in dataset.coords if k == "epoch" or k.endswith("_epoch")] - - for k in epoch_keys: - # ensure dtype is int64 and set writer hint for TT2000 - data_array = dataset.coords[k].astype(np.int64) - data_array.encoding["dtype"] = np.int64 - data_array.encoding["cdf_type"] = "CDF_TIME_TT2000" - dataset.coords[k] = data_array - # Create variables with fill values. for key in IALIRT_DIMS.keys(): dims = IALIRT_DIMS[key] From 9c182813513a6902d5c2ac1093f2539f7497082d Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 9 Jan 2026 18:32:45 -0700 Subject: [PATCH 37/42] update epoch --- .../config/imap_ialirt_l1_variable_attrs.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index c202df17a5..6bdf60a260 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -417,6 +417,7 @@ hit_e_a_side_low_en: CATDESC: Low energy (>0.5 MeV) electrons (A-side) FIELDNAM: Electrons low energy (A-side) LABLAXIS: A-side e- >0.5 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -427,6 +428,7 @@ hit_e_a_side_med_en: CATDESC: Medium energy (<1 MeV) electrons (A-side) FIELDNAM: Electrons medium energy (A-side) LABLAXIS: A-side e- <1 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -437,6 +439,7 @@ hit_e_a_side_high_en: CATDESC: High energy (>3 MeV) electrons (A-side) FIELDNAM: Electrons high energy (A-side) LABLAXIS: A-side e- >3 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -447,6 +450,7 @@ hit_e_b_side_low_en: CATDESC: Low energy (>0.5 MeV) electrons (B-side) FIELDNAM: Electrons low energy (B-side) LABLAXIS: B-side e- >0.5 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -457,6 +461,7 @@ hit_e_b_side_med_en: CATDESC: Medium energy (<1 MeV) electrons (B-side) FIELDNAM: Electrons medium energy LABLAXIS: B-side e- <1 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -467,6 +472,7 @@ hit_e_b_side_high_en: CATDESC: High energy (>3 MeV) electrons (B-side) FIELDNAM: Electrons high energy (B-side) LABLAXIS: B-side e- >3 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -477,6 +483,7 @@ hit_h_omni_low_en: CATDESC: Low energy (6 to 8 MeV) protons (omnidirectional) FIELDNAM: Protons 6 to 8 MeV (omni) LABLAXIS: Omni H 6 to 8 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -487,6 +494,7 @@ hit_h_omni_med_en: CATDESC: Medium energy (12 to 15 MeV) protons (omnidirectional) FIELDNAM: Protons 12 to 15 MeV (omni) LABLAXIS: Omni H 12 to 15 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -497,6 +505,7 @@ hit_h_a_side_high_en: CATDESC: High energy (>70 MeV) protons (A-side) FIELDNAM: Protons >70 MeV (A-side) LABLAXIS: A-side H >70 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -507,6 +516,7 @@ hit_h_b_side_high_en: CATDESC: High energy (>70 MeV) protons (B-side) FIELDNAM: Protons >70 MeV (B-side) LABLAXIS: B-side H >70 MeV + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -517,6 +527,7 @@ hit_he_omni_low_en: CATDESC: Low energy (6 to 8 MeV/nuc) helium (omnidirectional) FIELDNAM: Helium 6 to 8 MeV/nuc (omni) LABLAXIS: Omni He 6 to 8 MeV/nuc + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -527,6 +538,7 @@ hit_he_omni_high_en: CATDESC: High energy (15 to 70 MeV/nuc) helium (omnidirectional) FIELDNAM: Helium 15 to 70 MeV/nuc (omni) LABLAXIS: Omni He 15 to 70 MeV/nuc + DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 VALIDMAX: 1000000000 @@ -624,6 +636,7 @@ swapi_pseudo_proton_density: CATDESC: Pseudo density of solar wind protons FIELDNAM: SW proton pseudo density LABLAXIS: SW p pseudo N + DEPEND_0: swapi_epoch UNITS: 1/cm^3 VALIDMIN: 0.0 VALIDMAX: 1000.0 @@ -634,6 +647,7 @@ swapi_pseudo_proton_speed: CATDESC: Pseudo speed of solar wind protons in solar inertial frame FIELDNAM: SW proton pseudo V in solar inertial frame LABLAXIS: SW p pseudo V + DEPEND_0: swapi_epoch UNITS: km/sec VALIDMIN: 0.0 VALIDMAX: 10000.0 @@ -644,6 +658,7 @@ swapi_pseudo_proton_temperature: CATDESC: Pseudo temperature of solar wind protons in plasma frame FIELDNAM: SW proton pseudo temperature LABLAXIS: SW p pseudo T + DEPEND_0: swapi_epoch UNITS: Kelvin VALIDMIN: 0.0 VALIDMAX: 50000000.0 @@ -661,6 +676,7 @@ swe_normalized_counts: DEPEND_1: swe_electron_energy DISPLAY_TYPE: stack_plot LABLAXIS: Electrons + DEPEND_0: swe_epoch VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. swe_counterstreaming_electrons: @@ -668,6 +684,7 @@ swe_counterstreaming_electrons: CATDESC: Counterstreaming electrons flag (1=counterstreaming, 0=unidirectional) FIELDNAM: Counterstreaming e- (1=counterstreaming) LABLAXIS: Counterstreaming e- + DEPEND_0: swe_epoch UNITS: "1=counterstreaming, 0=unidirectional" VALIDMIN: 0 VALIDMAX: 1 From 6ee4e9786acaef7b093c9d1d8e449938c81b6ffe Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 14 Jan 2026 09:54:47 -0700 Subject: [PATCH 38/42] address andriy comments --- .../config/imap_ialirt_global_cdf_attrs.yaml | 2 +- .../config/imap_ialirt_l1_variable_attrs.yaml | 275 +++++++----------- imap_processing/ialirt/utils/constants.py | 14 +- imap_processing/ialirt/utils/create_xarray.py | 104 ++----- 4 files changed, 142 insertions(+), 253 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml index 69639fcac0..53c2d86368 100644 --- a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml @@ -9,7 +9,7 @@ instrument_base: &instrument_base HIT (High-energy Ion Telescope), MAG (Magnetometer), and SWAPI (Solar Wind and Pickup Ion). See https://imap.princeton.edu for more details. - Instrument_type: CoDICE - Particles (space), HIT - Particles (space), MAG - Magnetic Fields (space), SWAPI - Particles (space), SWE - Particles (space) + Instrument_type: Particles (space), Magnetic Fields (space), Plasma and Solar Wind (space), Ephemeris/Attitude/Ancillary imap_ialirt_l1_realtime: <<: *instrument_base diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 6bdf60a260..93e656b5b7 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -1,7 +1,62 @@ +# Epochs +instrument_epoch: &instrument_epoch + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + FILLVAL: -9223372036854775808 + FORMAT: I20 + VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) + VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + RESOLUTION: ' ' + DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" + dtype: int64 + +codice_hi_epoch: + <<: *instrument_epoch + CATDESC: CoDICE-Hi time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: CoDICE-Hi epoch + +codice_lo_epoch: + <<: *instrument_epoch + CATDESC: CoDICE-Lo time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: CoDICE-Lo epoch + +hit_epoch: + <<: *instrument_epoch + CATDESC: HIT time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: HIT epoch + +mag_epoch: + <<: *instrument_epoch + CATDESC: MAG time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: MAG epoch + +swapi_epoch: + <<: *instrument_epoch + CATDESC: SWAPI time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: SWAPI epoch + +swe_epoch: + <<: *instrument_epoch + CATDESC: SWE time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: SWE epoch + +ephemeris_epoch: + <<: *instrument_epoch + CATDESC: Ephemeris time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: Ephemeris epoch + # Defaults default_attrs: &default # Assumed values for all variable attrs unless overwritten - DEPEND_0: epoch + DEPEND_0: ephemeris_epoch DISPLAY_TYPE: time_series FILLVAL: -9223372036854775808 FORMAT: I12 @@ -66,91 +121,13 @@ default_float64_attrs: &default_float64 VALIDMAX: 1.7976931348623157e+308 dtype: float64 -# Epochs -instrument_epoch: &instrument_epoch - CATDESC: Time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: epoch - LABLAXIS: epoch - FILLVAL: -9223372036854775808 - FORMAT: I20 - VALIDMIN: 315576066184000000 # 2010-01-01T00:00:00 mission start (APL 0 epoch) - VALIDMAX: 3155716869184000000 # 2100-01-01T00:00:00 mission end - UNITS: ns - VAR_TYPE: support_data - SCALETYP: linear - MONOTON: INCREASE - TIME_BASE: J2000 - TIME_SCALE: Terrestrial Time - REFERENCE_POSITION: Rotating Earth Geoid - RESOLUTION: ' ' - DICT_KEY: "SPASE>Support>SupportQuantity:Temporal" - dtype: int64 - -codice_hi_epoch: - <<: *instrument_epoch - CATDESC: CoDICE-Hi time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: CoDICE-Hi epoch - -codice_lo_epoch: - <<: *instrument_epoch - CATDESC: CoDICE-Lo time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: CoDICE-Lo epoch - -hit_epoch: - <<: *instrument_epoch - CATDESC: HIT time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: HIT epoch - -mag_epoch: - <<: *instrument_epoch - CATDESC: MAG time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: MAG epoch - -swapi_epoch: - <<: *instrument_epoch - CATDESC: SWAPI time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: SWAPI epoch - -swe_epoch: - <<: *instrument_epoch - CATDESC: SWE time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: SWE epoch - -epoch: - <<: *instrument_epoch - CATDESC: Spacecraft time, number of nanoseconds since J2000 with leap seconds included - FIELDNAM: Spacecraft epoch - # Labels -B_GSM_component_index: - CATDESC: B GSM component index - FIELDNAM: B GSM component index - FORMAT: I2 - VAR_TYPE: support_data - UNITS: " " - VALIDMIN: 1 - VALIDMAX: 3 - dtype: int8 - LABL_PTR_1: B_GSM_labels - B_GSM_labels: CATDESC: B GSM component labels FIELDNAM: B GSM component labels FORMAT: A8 VAR_TYPE: metadata UNITS: " " - DEPEND_0: B_GSM_component_index - -B_GSE_component_index: - CATDESC: B GSE component index - FIELDNAM: B GSE component index - FORMAT: I2 - VAR_TYPE: support_data - UNITS: " " - VALIDMIN: 1 - VALIDMAX: 3 - dtype: int8 - LABL_PTR_1: B_GSE_labels B_GSE_labels: CATDESC: B GSE component labels @@ -158,18 +135,6 @@ B_GSE_labels: FORMAT: A8 VAR_TYPE: metadata UNITS: " " - DEPEND_0: B_GSE_component_index - -B_RTN_component_index: - CATDESC: B RTN component index - FIELDNAM: B RTN component index - FORMAT: I2 - VAR_TYPE: support_data - UNITS: " " - VALIDMIN: 1 - VALIDMAX: 3 - dtype: int8 - LABL_PTR_1: B_RTN_labels B_RTN_labels: CATDESC: B RTN component labels B radial (RTN), B tangential (RTN), B normal (RTN) @@ -177,40 +142,31 @@ B_RTN_labels: FORMAT: A18 VAR_TYPE: metadata UNITS: " " - DEPEND_0: B_RTN_component_index -sc_GSM_component_index: - CATDESC: Spacecraft GSM component index - FIELDNAM: Spacecraft GSM component index - FORMAT: I2 - VAR_TYPE: support_data +sc_GSM_position_labels: + CATDESC: Spacecraft position labels in GSM coordinates + FIELDNAM: Spacecraft GSM position labels + FORMAT: A8 + VAR_TYPE: metadata UNITS: " " - VALIDMIN: 1 - VALIDMAX: 3 - dtype: int8 - LABL_PTR_1: sc_GSM_labels -sc_GSM_labels: - CATDESC: Spacecraft position and velocity component labels in GSM coordinates - FIELDNAM: Spacecraft GSM component labels +sc_GSM_velocity_labels: + CATDESC: Spacecraft velocity labels in GSM coordinates + FIELDNAM: Spacecraft GSM velocity labels FORMAT: A8 VAR_TYPE: metadata UNITS: " " -sc_GSE_component_index: - CATDESC: Spacecraft GSE component index - FIELDNAM: Spacecraft GSE component index - FORMAT: I2 - VAR_TYPE: support_data +sc_GSE_position_labels: + CATDESC: Spacecraft position labels in GSE coordinates + FIELDNAM: Spacecraft GSE position labels + FORMAT: A8 + VAR_TYPE: metadata UNITS: " " - VALIDMIN: 1 - VALIDMAX: 3 - dtype: int8 - LABL_PTR_1: sc_GSE_labels -sc_GSE_labels: - CATDESC: Spacecraft position and velocity component labels in GSE coordinates - FIELDNAM: Spacecraft GSE component labels +sc_GSE_velocity_labels: + CATDESC: Spacecraft velocity labels in GSE coordinates + FIELDNAM: Spacecraft GSE velocity labels FORMAT: A8 VAR_TYPE: metadata UNITS: " " @@ -224,7 +180,6 @@ codice_hi_energy_center: VALIDMIN: 0.0 DELTA_PLUS_VAR: codice_hi_energy_plus DELTA_MINUS_VAR: codice_hi_energy_minus - LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 @@ -236,8 +191,6 @@ codice_hi_energy_minus: UNITS: "MeV" VALIDMAX: 20.0 VALIDMIN: 0.0 - DEPEND_0: codice_hi_energy_center - LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 @@ -249,20 +202,10 @@ codice_hi_energy_plus: UNITS: "MeV" VALIDMAX: 20.0 VALIDMIN: 0.0 - DEPEND_0: codice_hi_energy_center - LABL_PTR_1: codice_hi_energy_labels FILLVAL: -1.0e31 FORMAT: F12.6 dtype: float32 -codice_hi_energy_labels: - CATDESC: CoDICE-Hi energy channel labels - FIELDNAM: CoDICE-Hi energy channel labels - FORMAT: A12 - VAR_TYPE: metadata - UNITS: " " - DEPEND_0: codice_hi_energy_center - codice_hi_elevation: CATDESC: Elevation Angle FIELDNAM: Elevation Angle @@ -282,7 +225,6 @@ codice_hi_elevation_labels: FORMAT: A12 VAR_TYPE: metadata UNITS: " " - DEPEND_0: codice_hi_elevation codice_hi_spin_sector: CATDESC: CoDICE-Hi spin sector index @@ -293,7 +235,6 @@ codice_hi_spin_sector: VALIDMIN: 0 VALIDMAX: 3 dtype: int32 - LABL_PTR_1: codice_hi_spin_sector_labels codice_hi_spin_sector_labels: CATDESC: CoDICE-Hi spin sector labels @@ -301,7 +242,6 @@ codice_hi_spin_sector_labels: FORMAT: A8 VAR_TYPE: metadata UNITS: " " - DEPEND_0: codice_hi_spin_sector codice_hi_spin_angle: <<: *default_float32 @@ -332,7 +272,7 @@ swe_electron_energy: # Variables codice_hi_h: <<: *default_float32 - CATDESC: Proton intensity in 4 spin angles, 15 energy ranges, 4 spin sectors, and 4 elevations + CATDESC: Proton intensity in 4 epochs, 15 energy ranges, 4 spin sectors, and 4 elevations FIELDNAM: H+ intensity LABLAXIS: H+ intensity DEPEND_0: codice_hi_epoch @@ -344,7 +284,7 @@ codice_hi_h: DEPEND_3: codice_hi_elevation DISPLAY_TYPE: no_plot FORMAT: F16.6 - VAR_NOTES: Proton intensity for energy ranges (MeV) are 0.0200 to 0.0283, 0.0283 to 0.0400, 0.0400 to 0.0566, 0.0566 to 0.0800, 0.0800 to 0.113, 0.113 to 0.160, 0.160 to 0.226, 0.226 to 0.320, 0.320 to 0.453, 0.453 to 0.640, 0.640 to 0.905, 0.905 to 1.28, 1.28 to 1.81, 1.81 to 2.56, 2.56 to 3.62. + VAR_NOTES: Energy ranges (MeV) are 0.0200 to 0.0283, 0.0283 to 0.0400, 0.0400 to 0.0566, 0.0566 to 0.0800, 0.0800 to 0.113, 0.113 to 0.160, 0.160 to 0.226, 0.226 to 0.320, 0.320 to 0.453, 0.453 to 0.640, 0.640 to 0.905, 0.905 to 1.28, 1.28 to 1.81, 1.81 to 2.56, 2.56 to 3.62. codice_lo_c_over_o_abundance: <<: *default_float32 @@ -414,8 +354,8 @@ codice_lo_fe_low_over_fe_high: hit_e_a_side_low_en: <<: *default_uint32 - CATDESC: Low energy (>0.5 MeV) electrons (A-side) - FIELDNAM: Electrons low energy (A-side) + CATDESC: Low energy (>0.5 MeV) electron count rate (A-side, anti-sunward) + FIELDNAM: Low energy electron count rate (A-side) LABLAXIS: A-side e- >0.5 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -425,8 +365,8 @@ hit_e_a_side_low_en: hit_e_a_side_med_en: <<: *default_uint32 - CATDESC: Medium energy (<1 MeV) electrons (A-side) - FIELDNAM: Electrons medium energy (A-side) + CATDESC: Medium energy (<1 MeV) electron count rate (A-side, anti-sunward) + FIELDNAM: Medium energy electron count rate (A-side) LABLAXIS: A-side e- <1 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -436,8 +376,8 @@ hit_e_a_side_med_en: hit_e_a_side_high_en: <<: *default_uint32 - CATDESC: High energy (>3 MeV) electrons (A-side) - FIELDNAM: Electrons high energy (A-side) + CATDESC: High energy (>3 MeV) electron count rate (A-side, anti-sunward) + FIELDNAM: High energy electron count rate (A-side) LABLAXIS: A-side e- >3 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -447,8 +387,8 @@ hit_e_a_side_high_en: hit_e_b_side_low_en: <<: *default_uint32 - CATDESC: Low energy (>0.5 MeV) electrons (B-side) - FIELDNAM: Electrons low energy (B-side) + CATDESC: Low energy (>0.5 MeV) electron count rate (B-side, sunward) + FIELDNAM: Low energy electron count rate (B-side) LABLAXIS: B-side e- >0.5 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -458,8 +398,8 @@ hit_e_b_side_low_en: hit_e_b_side_med_en: <<: *default_uint32 - CATDESC: Medium energy (<1 MeV) electrons (B-side) - FIELDNAM: Electrons medium energy + CATDESC: Medium energy (<1 MeV) electron count rate (B-side, sunward) + FIELDNAM: Medium energy electron count rate (B-side) LABLAXIS: B-side e- <1 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -469,8 +409,8 @@ hit_e_b_side_med_en: hit_e_b_side_high_en: <<: *default_uint32 - CATDESC: High energy (>3 MeV) electrons (B-side) - FIELDNAM: Electrons high energy (B-side) + CATDESC: High energy (>3 MeV) electron count rate (B-side, sunward) + FIELDNAM: High energy electron count rate (B-side) LABLAXIS: B-side e- >3 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -480,9 +420,9 @@ hit_e_b_side_high_en: hit_h_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6 to 8 MeV) protons (omnidirectional) - FIELDNAM: Protons 6 to 8 MeV (omni) - LABLAXIS: Omni H 6 to 8 MeV + CATDESC: Low energy (6 to 8 MeV) proton count rate (omnidirectional) + FIELDNAM: Proton count rate 6 to 8 MeV (omni) + LABLAXIS: Omni H+ 6 to 8 MeV DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 @@ -491,9 +431,9 @@ hit_h_omni_low_en: hit_h_omni_med_en: <<: *default_uint32 - CATDESC: Medium energy (12 to 15 MeV) protons (omnidirectional) - FIELDNAM: Protons 12 to 15 MeV (omni) - LABLAXIS: Omni H 12 to 15 MeV + CATDESC: Medium energy (12 to 15 MeV) proton count rate (omnidirectional) + FIELDNAM: Proton count rate 12 to 15 MeV (omni) + LABLAXIS: Omni H+ 12 to 15 MeV DEPEND_0: hit_epoch UNITS: counts per second VALIDMIN: 0 @@ -502,8 +442,8 @@ hit_h_omni_med_en: hit_h_a_side_high_en: <<: *default_uint32 - CATDESC: High energy (>70 MeV) protons (A-side) - FIELDNAM: Protons >70 MeV (A-side) + CATDESC: High energy (>70 MeV) proton count rate (A-side, anti-sunward) + FIELDNAM: Proton count rate >70 MeV (A-side) LABLAXIS: A-side H >70 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -513,8 +453,8 @@ hit_h_a_side_high_en: hit_h_b_side_high_en: <<: *default_uint32 - CATDESC: High energy (>70 MeV) protons (B-side) - FIELDNAM: Protons >70 MeV (B-side) + CATDESC: High energy (>70 MeV) proton count rate (B-side, sunward) + FIELDNAM: Proton count rate >70 MeV (B-side) LABLAXIS: B-side H >70 MeV DEPEND_0: hit_epoch UNITS: counts per second @@ -524,8 +464,8 @@ hit_h_b_side_high_en: hit_he_omni_low_en: <<: *default_uint32 - CATDESC: Low energy (6 to 8 MeV/nuc) helium (omnidirectional) - FIELDNAM: Helium 6 to 8 MeV/nuc (omni) + CATDESC: Low energy (6 to 8 MeV/nuc) helium count rate (omnidirectional) + FIELDNAM: Helium count rate 6 to 8 MeV/nuc (omni) LABLAXIS: Omni He 6 to 8 MeV/nuc DEPEND_0: hit_epoch UNITS: counts per second @@ -535,8 +475,8 @@ hit_he_omni_low_en: hit_he_omni_high_en: <<: *default_uint32 - CATDESC: High energy (15 to 70 MeV/nuc) helium (omnidirectional) - FIELDNAM: Helium 15 to 70 MeV/nuc (omni) + CATDESC: High energy (15 to 70 MeV/nuc) helium count rate (omnidirectional) + FIELDNAM: Helium count rate 15 to 70 MeV/nuc (omni) LABLAXIS: Omni He 15 to 70 MeV/nuc DEPEND_0: hit_epoch UNITS: counts per second @@ -600,7 +540,6 @@ mag_B_GSE: FIELDNAM: B vector in GSE coordinates LABLAXIS: B (GSE) DEPEND_0: mag_epoch - DEPEND_1: B_GSE_component_index LABL_PTR_1: B_GSE_labels UNITS: nT FORMAT: F13.6 @@ -670,12 +609,12 @@ swe_normalized_counts: <<: *default_uint32 CATDESC: Normalized electron counts FIELDNAM: Normalized electron counts - UNITS: Normalized counts + UNITS: normalized counts VALIDMIN: 0 VALIDMAX: 500000 DEPEND_1: swe_electron_energy DISPLAY_TYPE: stack_plot - LABLAXIS: Electrons + LABLAXIS: Electron counts DEPEND_0: swe_epoch VAR_NOTES: The nominal central energies for the 8 i-ALiRT channels are 100.4, 140, 194, 270, 376, 523, 727, and 1011 eV. @@ -685,7 +624,7 @@ swe_counterstreaming_electrons: FIELDNAM: Counterstreaming e- (1=counterstreaming) LABLAXIS: Counterstreaming e- DEPEND_0: swe_epoch - UNITS: "1=counterstreaming, 0=unidirectional" + UNITS: "1=counterstreaming" VALIDMIN: 0 VALIDMAX: 1 VAR_NOTES: A value of 1 indicates that counterstreaming electrons were observed, a value of 0 indicates unidirectional electron flow. @@ -696,7 +635,7 @@ sc_position_GSE: FIELDNAM: Spacecraft position in GSE LABLAXIS: SC position (GSE) UNITS: km - LABL_PTR_1: sc_GSE_labels + LABL_PTR_1: sc_GSE_position_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 FORMAT: F15.6 @@ -707,7 +646,7 @@ sc_velocity_GSE: FIELDNAM: Spacecraft velocity in GSE LABLAXIS: SC velocity (GSE) UNITS: km/s - LABL_PTR_1: sc_GSE_labels + LABL_PTR_1: sc_GSE_velocity_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 @@ -717,7 +656,7 @@ sc_position_GSM: FIELDNAM: Spacecraft position in GSM coordinates LABLAXIS: SC position (GSM) UNITS: km - LABL_PTR_1: sc_GSM_labels + LABL_PTR_1: sc_GSM_position_labels VALIDMIN: -3000000.0 VALIDMAX: 3000000.0 FORMAT: F15.6 @@ -728,6 +667,6 @@ sc_velocity_GSM: FIELDNAM: Spacecraft velocity in GSM LABLAXIS: SC velocity (GSM) UNITS: km/s - LABL_PTR_1: sc_GSM_labels + LABL_PTR_1: sc_GSM_velocity_labels VALIDMIN: -1000.0 VALIDMAX: 1000.0 diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 2b3bd16309..5e09e61777 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -49,11 +49,11 @@ # Azimuth angle (φ) of the magnetic field in GSM coordinates "mag_phi_B_GSM": ["mag_epoch"], # Magnetic field vector in GSE coordinates - "mag_B_GSE": ["mag_epoch", "B_GSE_component_index"], + "mag_B_GSE": ["mag_epoch", "B_GSE_labels"], # Magnetic field vector in GSM coordinates - "mag_B_GSM": ["mag_epoch", "B_GSM_component_index"], + "mag_B_GSM": ["mag_epoch", "B_GSM_labels"], # Magnetic field vector in RTN coordinates - "mag_B_RTN": ["mag_epoch", "B_RTN_component_index"], + "mag_B_RTN": ["mag_epoch", "B_RTN_labels"], # Pseudo density of solar wind protons "swapi_pseudo_proton_density": ["swapi_epoch"], # Pseudo speed of solar wind protons in solar inertial frame @@ -65,13 +65,13 @@ # SWE Counterstreaming flag "swe_counterstreaming_electrons": ["swe_epoch"], # Spacecraft position in GSE coordinates - "sc_position_GSE": ["epoch", "sc_GSE_component_index"], + "sc_position_GSE": ["ephemeris_epoch", "sc_GSE_position_labels"], # Spacecraft velocity in GSE coordinates - "sc_velocity_GSE": ["epoch", "sc_GSE_component_index"], + "sc_velocity_GSE": ["ephemeris_epoch", "sc_GSE_velocity_labels"], # Spacecraft position in GSM coordinates - "sc_position_GSM": ["epoch", "sc_GSM_component_index"], + "sc_position_GSM": ["ephemeris_epoch", "sc_GSM_position_labels"], # Spacecraft velocity in GSM coordinates - "sc_velocity_GSM": ["epoch", "sc_GSM_component_index"], + "sc_velocity_GSM": ["ephemeris_epoch", "sc_GSM_velocity_labels"], } IALIRT_DTYPES = { diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index e0edd5a87a..7b77fef1a5 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -69,88 +69,51 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 epoch_arrays[coord] = xr.DataArray( data=np.array(arr, dtype=np.int64), name=coord, dims=[coord], attrs=attr ) - # numeric component index for GSE vector (1-based; use 0-based if you prefer) - sc_gsm_component_index = xr.DataArray( - data=np.arange(1, 4, dtype=np.int32), - name="sc_GSM_component_index", - dims=["sc_GSM_component_index"], - attrs=cdf_manager.get_variable_attributes( - "sc_GSM_component_index", check_schema=False - ), - ) - sc_gsm_component = xr.DataArray( - ["x (GSM)", "y (GSM)", "z (GSM)"], - name="sc_GSM_labels", - dims=["sc_GSM_labels"], - attrs=cdf_manager.get_variable_attributes("sc_GSM_labels", check_schema=False), + sc_gsm_position_component = xr.DataArray( + ["sc X (GSM)", "sc Y (GSM)", "sc Z (GSM)"], + name="sc_GSM_position_labels", + dims=["sc_GSM_position_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSM_position_labels", check_schema=False), ) - - # numeric component index for GSE vector (1-based; use 0-based if you prefer) - sc_gse_component_index = xr.DataArray( - data=np.arange(1, 4, dtype=np.int32), - name="sc_GSE_component_index", - dims=["sc_GSE_component_index"], - attrs=cdf_manager.get_variable_attributes( - "sc_GSE_component_index", check_schema=False - ), + sc_gsm_velocity_component = xr.DataArray( + ["sc Vx (GSM)", "sc Vy (GSM)", "sc Vz (GSM)"], + name="sc_GSM_velocity_labels", + dims=["sc_GSM_velocity_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSM_velocity_labels", check_schema=False), ) - sc_gse_component = xr.DataArray( - ["x (GSE)", "y (GSE)", "z (GSE)"], - name="sc_GSE_labels", - dims=["sc_GSE_component_index"], - attrs=cdf_manager.get_variable_attributes("sc_GSE_labels", check_schema=False), + sc_gse_position_component = xr.DataArray( + ["sc X (GSE)", "sc Y (GSE)", "sc Z (GSE)"], + name="sc_GSE_position_labels", + dims=["sc_GSE_position_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSE_position_labels", check_schema=False), ) - - # numeric component index for GSE vector (1-based; use 0-based if you prefer) - gsm_component_index = xr.DataArray( - data=np.arange(1, 4, dtype=np.int32), - name="B_GSM_component_index", - dims=["B_GSM_component_index"], - attrs=cdf_manager.get_variable_attributes( - "B_GSM_component_index", check_schema=False - ), + sc_gse_velocity_component = xr.DataArray( + ["sc Vx (GSE)", "sc Vy (GSE)", "sc Vz (GSE)"], + name="sc_GSE_velocity_labels", + dims=["sc_GSE_velocity_labels"], + attrs=cdf_manager.get_variable_attributes("sc_GSE_velocity_labels", check_schema=False), ) gsm_component = xr.DataArray( ["Bx (GSM)", "By (GSM)", "Bz (GSM)"], name="B_GSM_labels", - dims=["B_GSM_component_index"], + dims=["B_GSM_labels"], attrs=cdf_manager.get_variable_attributes("B_GSM_labels", check_schema=False), ) - # numeric component index for GSE vector (1-based; use 0-based if you prefer) - gse_component_index = xr.DataArray( - data=np.arange(1, 4, dtype=np.int32), - name="B_GSE_component_index", - dims=["B_GSE_component_index"], - attrs=cdf_manager.get_variable_attributes( - "B_GSE_component_index", check_schema=False - ), - ) - gse_component = xr.DataArray( ["Bx (GSE)", "By (GSE)", "Bz (GSE)"], name="B_GSE_labels", - dims=["B_GSE_component_index"], + dims=["B_GSE_labels"], attrs=cdf_manager.get_variable_attributes("B_GSE_labels", check_schema=False), ) - # numeric component index for GSE vector (1-based; use 0-based if you prefer) - rtn_component_index = xr.DataArray( - data=np.arange(1, 4, dtype=np.int32), - name="B_RTN_component_index", - dims=["B_RTN_component_index"], - attrs=cdf_manager.get_variable_attributes( - "B_RTN_component_index", check_schema=False - ), - ) - rtn_component = xr.DataArray( ["B radial (RTN)", "B tangential (RTN)", "B normal (RTN)"], name="B_RTN_labels", - dims=["B_RTN_component_index"], + dims=["B_RTN_labels"], attrs=cdf_manager.get_variable_attributes("B_RTN_labels", check_schema=False), ) @@ -190,15 +153,6 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - codice_energy_labels = xr.DataArray( - [f"ch{n + 1}" for n in range(len(codice_hi_energy_center))], - name="codice_hi_energy_labels", - dims=["codice_hi_energy_center"], - attrs=cdf_manager.get_variable_attributes( - "codice_hi_energy_labels", check_schema=False - ), - ) - elevation = xr.DataArray( np.float32(HI_IALIRT_ELEVATION_ANGLE), name="codice_hi_elevation", @@ -266,20 +220,16 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "swapi_epoch": epoch_arrays["swapi_epoch"], "swe_epoch": epoch_arrays["swe_epoch"], "epoch": epoch_arrays["epoch"], - "B_GSM_component_index": gsm_component_index, "B_GSM_labels": gsm_component, - "B_GSE_component_index": gse_component_index, "B_GSE_labels": gse_component, - "B_RTN_component_index": rtn_component_index, "B_RTN_labels": rtn_component, - "sc_GSM_component_index": sc_gsm_component_index, - "sc_GSM_labels": sc_gsm_component, - "sc_GSE_component_index": sc_gse_component_index, - "sc_GSE_labels": sc_gse_component, + "sc_GSM_position_labels": sc_gsm_position_component, + "sc_GSM_velocity_labels": sc_gsm_velocity_component, + "sc_GSE_position_labels": sc_gse_position_component, + "sc_GSE_velocity_labels": sc_gse_velocity_component, "codice_hi_energy_center": codice_hi_energy_centers, "codice_hi_energy_minus": codice_energy_minus, "codice_hi_energy_plus": codice_energy_plus, - "codice_hi_energy_labels": codice_energy_labels, "codice_hi_elevation": elevation, "codice_hi_elevation_labels": elevation_labels, "codice_hi_spin_angle": spin_angle, From 5b8c7cfe54768f815a9ccdf4498a8367c519b9d0 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 14 Jan 2026 10:17:33 -0700 Subject: [PATCH 39/42] update tests --- imap_processing/ialirt/utils/create_xarray.py | 7 ++++--- imap_processing/tests/ialirt/unit/test_create_xarray.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 7b77fef1a5..c31e9a0c4e 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -19,6 +19,7 @@ hit_restricted_fields, swe_energy, ) +from imap_processing.spice.time import TTJ2000_EPOCH def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0912 @@ -59,8 +60,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for inst, arr in epochs.items(): if inst == "spacecraft": - coord = "epoch" - attr = cdf_manager.get_variable_attributes("epoch", check_schema=False) + coord = "ephemeris_epoch" + attr = cdf_manager.get_variable_attributes("ephemeris_epoch", check_schema=False) else: coord = f"{inst}_epoch" attr = cdf_manager.get_variable_attributes( @@ -219,7 +220,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 "mag_epoch": epoch_arrays["mag_epoch"], "swapi_epoch": epoch_arrays["swapi_epoch"], "swe_epoch": epoch_arrays["swe_epoch"], - "epoch": epoch_arrays["epoch"], + "ephemeris_epoch": epoch_arrays["ephemeris_epoch"], "B_GSM_labels": gsm_component, "B_GSE_labels": gse_component, "B_RTN_labels": rtn_component, diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 8e3da607f2..5e8c3d65bf 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -153,7 +153,7 @@ def test_create_dataset(): dataset["codice_hi_h"].isel(codice_hi_epoch=0).values, expected_zeros ) - assert dataset["mag_B_GSE"].dims == ("mag_epoch", "B_GSE_component_index") + assert dataset["mag_B_GSE"].dims == ("mag_epoch", "B_GSE_labels") assert dataset["swe_normalized_counts"].dims == ("swe_epoch", "swe_electron_energy") assert dataset["codice_hi_h"].dims == ( "codice_hi_epoch", @@ -164,6 +164,7 @@ def test_create_dataset(): # Tests that you can write to a cdf. dataset.attrs["Data_version"] = "001" + dataset.attrs["Start_date"] = "20260114" test_data_path = write_cdf(dataset, istp=True, compression=None) assert test_data_path.exists() From 3e009ec8dd57b889494765593a2be202bf5b6814 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 16 Jan 2026 11:54:24 -0700 Subject: [PATCH 40/42] simple pr responses --- .../cdf/config/imap_ialirt_global_cdf_attrs.yaml | 2 +- .../cdf/config/imap_ialirt_l1_variable_attrs.yaml | 14 +++++++------- imap_processing/codice/constants.py | 2 +- imap_processing/ialirt/l0/parse_mag.py | 2 +- imap_processing/ialirt/l0/process_codice.py | 8 ++++---- imap_processing/ialirt/utils/create_xarray.py | 14 +++++++------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml index 53c2d86368..bab66ac8a7 100644 --- a/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_global_cdf_attrs.yaml @@ -15,4 +15,4 @@ imap_ialirt_l1_realtime: <<: *instrument_base Data_type: L1_REALTIME>Level-1 Realtime Logical_source: imap_ialirt_l1_realtime - Logical_source_description: IMAP Active Link for Real-Time (I-ALiRT) Level-1 Realtime Data. + Logical_source_description: IMAP Active Link for Real-Time (I-ALiRT) Level-1 Data. diff --git a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml index 93e656b5b7..bfa9083a81 100644 --- a/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml @@ -20,37 +20,37 @@ instrument_epoch: &instrument_epoch codice_hi_epoch: <<: *instrument_epoch - CATDESC: CoDICE-Hi time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time CoDICE-Hi instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: CoDICE-Hi epoch codice_lo_epoch: <<: *instrument_epoch - CATDESC: CoDICE-Lo time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time CoDICE-Lo instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: CoDICE-Lo epoch hit_epoch: <<: *instrument_epoch - CATDESC: HIT time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time HIT instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: HIT epoch mag_epoch: <<: *instrument_epoch - CATDESC: MAG time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time MAG instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: MAG epoch swapi_epoch: <<: *instrument_epoch - CATDESC: SWAPI time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time SWAPI instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: SWAPI epoch swe_epoch: <<: *instrument_epoch - CATDESC: SWE time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time SWE instrument took the measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: SWE epoch ephemeris_epoch: <<: *instrument_epoch - CATDESC: Ephemeris time, number of nanoseconds since J2000 with leap seconds included + CATDESC: Time of ephemeris measurement, number of nanoseconds since J2000 with leap seconds included FIELDNAM: Ephemeris epoch # Defaults diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index 467e46198e..670a1873ea 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -83,7 +83,7 @@ 47.1, 114.3, ], - dtype=float, + dtype=np.float32, ) HI_IALIRT_REF_SPIN_ANGLE = np.array( [ diff --git a/imap_processing/ialirt/l0/parse_mag.py b/imap_processing/ialirt/l0/parse_mag.py index f715d0b972..7eacfa2b1a 100644 --- a/imap_processing/ialirt/l0/parse_mag.py +++ b/imap_processing/ialirt/l0/parse_mag.py @@ -717,7 +717,7 @@ def process_packet( met = grouped_data["met"][(grouped_data["group"] == group).values] # Calculate the center of the time variable - sc_met = int(np.mean([np.min(met), np.max(met)])) + sc_met = (met[0] + met[-1]) // 2 met_all.append(sc_met) mago_times_all.append(time_data["primary_epoch"]) mago_vectors_all.append(mago_inertial_vector) diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index b9f01a07cd..4cd9bd0b64 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -472,12 +472,12 @@ def process_codice( l2_lo = calculate_ratios(l1b_lo, l2_lut_path, l2_geometric_factor_path) # Calculate the center of the time variable - sc_met = int(np.mean([np.min(met), np.max(met)])) + sc_met = (met[0] + met[-1]) // 2 codice_lo_data.append( { "apid": 478, - "met": int(sc_met), + "met": sc_met, "met_in_utc": met_to_utc(sc_met).split(".")[0], "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", @@ -517,12 +517,12 @@ def process_codice( ).tolist() # Calculate the center of the time variable - sc_met = int(np.mean([np.min(met), np.max(met)])) + sc_met = (met[0] + met[-1]) // 2 codice_hi_data.append( { "apid": 478, - "met": int(sc_met), + "met": sc_met, "met_in_utc": met_to_utc(sc_met).split(".")[0], "ttj2000ns": int(met_to_ttj2000ns(sc_met)), "instrument": f"{sensor}", diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index c31e9a0c4e..a213850ddd 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -155,7 +155,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) elevation = xr.DataArray( - np.float32(HI_IALIRT_ELEVATION_ANGLE), + HI_IALIRT_ELEVATION_ANGLE, name="codice_hi_elevation", dims=["codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( @@ -201,11 +201,11 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 spin_sector_labels = xr.DataArray( [ - "sector_0", - "sector_1", - "sector_2", - "sector_3", - ], # or ["sector0","sector1",...] - whatever you prefer + "0", + "1", + "2", + "3", + ], name="codice_hi_spin_sector_labels", dims=["codice_hi_spin_sector"], attrs=cdf_manager.get_variable_attributes( @@ -244,7 +244,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ) # Create variables with fill values. - for key in IALIRT_DIMS.keys(): + for key in IALIRT_DIMS: dims = IALIRT_DIMS[key] attrs = cdf_manager.get_variable_attributes(key, check_schema=False) fill = attrs["FILLVAL"] From a2999b50986c61dc135c26a8d9722af6d9b53cca Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 16 Jan 2026 13:39:45 -0700 Subject: [PATCH 41/42] formatting --- imap_processing/ialirt/l0/parse_mag.py | 13 ++++------- imap_processing/ialirt/l0/process_codice.py | 20 +++------------- imap_processing/ialirt/l0/process_hit.py | 11 ++------- imap_processing/ialirt/l0/process_swapi.py | 10 ++------ imap_processing/ialirt/l0/process_swe.py | 19 +++------------ imap_processing/ialirt/utils/constants.py | 15 ++++++++++++ imap_processing/ialirt/utils/create_xarray.py | 14 +++-------- imap_processing/ialirt/utils/grouping.py | 23 +++++++++++++++++++ .../tests/ialirt/unit/test_create_xarray.py | 14 +++++++++++ .../tests/ialirt/unit/test_grouping.py | 15 +++++++++++- 10 files changed, 83 insertions(+), 71 deletions(-) diff --git a/imap_processing/ialirt/l0/parse_mag.py b/imap_processing/ialirt/l0/parse_mag.py index 7eacfa2b1a..9a1e750142 100644 --- a/imap_processing/ialirt/l0/parse_mag.py +++ b/imap_processing/ialirt/l0/parse_mag.py @@ -16,7 +16,7 @@ Packet2, Packet3, ) -from imap_processing.ialirt.utils.grouping import find_groups +from imap_processing.ialirt.utils.grouping import find_groups, _populate_instrument_header_items from imap_processing.ialirt.utils.time import calculate_time from imap_processing.mag.l1a.mag_l1a_data import TimeTuple from imap_processing.mag.l1b.mag_l1b import ( @@ -716,9 +716,7 @@ def process_packet( ) met = grouped_data["met"][(grouped_data["group"] == group).values] - # Calculate the center of the time variable - sc_met = (met[0] + met[-1]) // 2 - met_all.append(sc_met) + met_all.append(met) mago_times_all.append(time_data["primary_epoch"]) mago_vectors_all.append(mago_inertial_vector) magi_vectors_all.append(magi_inertial_vector) @@ -757,11 +755,8 @@ def process_packet( continue mag_data.append( - { - "apid": 478, - "met": int(met_all[i]), - "met_in_utc": met_to_utc(met_all[i]).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(met_all[i])), + _populate_instrument_header_items(met_all[i]) | + { "instrument": "mag", "mag_epoch": int(mago_times_all[i]), "mag_B_GSE": [Decimal(f"{v:.3f}") for v in gse_vector[i]], diff --git a/imap_processing/ialirt/l0/process_codice.py b/imap_processing/ialirt/l0/process_codice.py index 4cd9bd0b64..1372478c8e 100644 --- a/imap_processing/ialirt/l0/process_codice.py +++ b/imap_processing/ialirt/l0/process_codice.py @@ -21,7 +21,7 @@ get_geometric_factor_lut, process_lo_species_intensity, ) -from imap_processing.ialirt.utils.grouping import find_groups +from imap_processing.ialirt.utils.grouping import find_groups, _populate_instrument_header_items from imap_processing.ialirt.utils.time import calculate_time from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc @@ -471,15 +471,8 @@ def process_codice( ) l2_lo = calculate_ratios(l1b_lo, l2_lut_path, l2_geometric_factor_path) - # Calculate the center of the time variable - sc_met = (met[0] + met[-1]) // 2 - codice_lo_data.append( - { - "apid": 478, - "met": sc_met, - "met_in_utc": met_to_utc(sc_met).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met)), + _populate_instrument_header_items(met) | { "instrument": f"{sensor}", f"{sensor}_c_over_o_abundance": l2_lo.c_over_o_abundance, f"{sensor}_mg_over_o_abundance": l2_lo.mg_over_o_abundance, @@ -516,15 +509,8 @@ def process_codice( l2_hi ).tolist() - # Calculate the center of the time variable - sc_met = (met[0] + met[-1]) // 2 - codice_hi_data.append( - { - "apid": 478, - "met": sc_met, - "met_in_utc": met_to_utc(sc_met).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met)), + _populate_instrument_header_items(met) | { "instrument": f"{sensor}", f"{sensor}_epoch": [int(epoch) for epoch in l1b_hi["epoch"]], f"{sensor}_h": dec_l2_hi, diff --git a/imap_processing/ialirt/l0/process_hit.py b/imap_processing/ialirt/l0/process_hit.py index baa68253fb..36143b01d9 100644 --- a/imap_processing/ialirt/l0/process_hit.py +++ b/imap_processing/ialirt/l0/process_hit.py @@ -5,7 +5,7 @@ import numpy as np import xarray as xr -from imap_processing.ialirt.utils.grouping import find_groups +from imap_processing.ialirt.utils.grouping import find_groups, _populate_instrument_header_items from imap_processing.ialirt.utils.time import calculate_time from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc @@ -162,17 +162,10 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: (grouped_data["group"] == group).values ] met = grouped_data["met"][(grouped_data["group"] == group).values] - # Calculate the center of the time variable - sc_met = int(np.mean([np.min(met), np.max(met)])) - l1 = create_l1(fast_rate_1, fast_rate_2, slow_rate) hit_data.append( - { - "apid": 478, - "met": sc_met, - "met_in_utc": met_to_utc(sc_met).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met)), + _populate_instrument_header_items(met) | { "instrument": "hit", "hit_e_a_side_low_en": int(l1["IALRT_RATE_1"] + l1["IALRT_RATE_2"]), "hit_e_a_side_med_en": int(l1["IALRT_RATE_5"] + l1["IALRT_RATE_6"]), diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 175063ef49..e8fb070584 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -10,7 +10,7 @@ from scipy.special import erf from imap_processing.ialirt.constants import IalirtSwapiConstants as Consts -from imap_processing.ialirt.utils.grouping import find_groups +from imap_processing.ialirt.utils.grouping import find_groups, _populate_instrument_header_items from imap_processing.ialirt.utils.time import calculate_time from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc from imap_processing.swapi.l1.swapi_l1 import process_sweep_data @@ -163,8 +163,6 @@ def process_swapi_ialirt( ] met = grouped_dataset["met"][(grouped_dataset["group"] == group).values] - # Calculate the center of the time variable - sc_met = int(np.mean([np.min(met), np.max(met)])) # Ensure no duplicates and all values from 0 to 11 are present if not np.array_equal(seq_values.values.astype(int), np.arange(12)): @@ -206,11 +204,7 @@ def process_swapi_ialirt( ) swapi_data.append( - { - "apid": 478, - "met": sc_met, - "met_in_utc": met_to_utc(sc_met).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met)), + _populate_instrument_header_items(met) | { "instrument": "swapi", "swapi_pseudo_proton_speed": Decimal(f"{pseudo_speed:.3f}"), "swapi_pseudo_proton_density": Decimal(f"{pseudo_density:.3f}"), diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index f78513b711..29b10fea4f 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -7,7 +7,7 @@ import xarray as xr from numpy.typing import NDArray -from imap_processing.ialirt.utils.grouping import find_groups +from imap_processing.ialirt.utils.grouping import find_groups, _populate_instrument_header_items from imap_processing.ialirt.utils.time import calculate_time from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc from imap_processing.swe.l1a.swe_science import decompressed_counts @@ -549,33 +549,20 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list summed_second = normalized_second_half.sum(axis=(1, 2)) met_first_half = grouped["met"].where(grouped["swe_seq"] == 0, drop=True).values - sc_met_first_half = int( - np.mean([np.min(met_first_half), np.max(met_first_half)]) - ) met_second_half = ( grouped["met"].where(grouped["swe_seq"] == 30, drop=True).values ) - sc_met_second_half = int( - np.mean([np.min(met_second_half), np.max(met_second_half)]) - ) swe_data.append( - { - "apid": 478, - "met": sc_met_first_half, - "met_in_utc": met_to_utc(sc_met_first_half).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met_first_half)), + _populate_instrument_header_items(met_first_half) |{ "instrument": "swe", "swe_normalized_counts": [int(val) for val in summed_first], "swe_counterstreaming_electrons": bde_first_half, }, ) swe_data.append( + _populate_instrument_header_items(met_second_half) | { - "apid": 478, - "met": sc_met_second_half, - "met_in_utc": met_to_utc(sc_met_second_half).split(".")[0], - "ttj2000ns": int(met_to_ttj2000ns(sc_met_second_half)), "instrument": "swe", "swe_normalized_counts": [int(val) for val in summed_second], "swe_counterstreaming_electrons": bde_second_half, diff --git a/imap_processing/ialirt/utils/constants.py b/imap_processing/ialirt/utils/constants.py index 5e09e61777..295a8da5a9 100644 --- a/imap_processing/ialirt/utils/constants.py +++ b/imap_processing/ialirt/utils/constants.py @@ -2,6 +2,11 @@ import numpy as np +from imap_processing.codice.constants import ( + HI_IALIRT_ELEVATION_ANGLE, + HI_IALIRT_REF_SPIN_ANGLE, +) + IALIRT_DIMS = { # H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins "codice_hi_h": [ @@ -176,6 +181,16 @@ 0.57601651, ] +# Calculate spin angle +# Formula: +# θ_(g,n) = (θ_(g,0)+90°* n) mod 360° +# where +# n is number of sectored angles, 0 to 3, +# g is size of the group (inst_az), 0 to 3, +HI_IALIRT_SPIN_ANGLE = ( + HI_IALIRT_REF_SPIN_ANGLE[:, np.newaxis] + np.array([0, 1, 2, 3]) * 90 +) % 360.0 + swe_energy = [ 100.4, 140.0, diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index a213850ddd..c97656300e 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -16,6 +16,7 @@ codice_hi_energy_center, codice_hi_energy_minus, codice_hi_energy_plus, + HI_IALIRT_SPIN_ANGLE, hit_restricted_fields, swe_energy, ) @@ -172,17 +173,8 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 ), ) - # Calculate spin angle - # Formula: - # θ_(g,n) = (θ_(g,0)+90°* n) mod 360° - # where - # n is number of sectored angles, 0 to 3, - # g is size of the group (inst_az), 0 to 3, - spin_angles = ( - HI_IALIRT_REF_SPIN_ANGLE[:, np.newaxis] + np.array([0, 1, 2, 3]) * 90 - ) % 360.0 spin_angle = xr.DataArray( - data=np.float32(spin_angles), + data=HI_IALIRT_SPIN_ANGLE.astype(np.float32), name="codice_hi_spin_angle", dims=["codice_hi_spin_sector", "codice_hi_elevation"], attrs=cdf_manager.get_variable_attributes( @@ -258,7 +250,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 for i, record in enumerate(by_inst.get("mag", [])): for key in IALIRT_DIMS.keys(): if key in ["mag_B_GSE", "mag_B_GSM", "mag_B_RTN"]: - dataset[key].data[i, :] = np.asarray(record[key], dtype=np.float32) + dataset[key].data[i, :] = record[key] if key in [ "mag_B_magnitude", diff --git a/imap_processing/ialirt/utils/grouping.py b/imap_processing/ialirt/utils/grouping.py index ca0ae6ac28..6cdad041b2 100644 --- a/imap_processing/ialirt/utils/grouping.py +++ b/imap_processing/ialirt/utils/grouping.py @@ -4,6 +4,7 @@ import numpy as np import xarray as xr +from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc logger = logging.getLogger(__name__) @@ -124,3 +125,25 @@ def find_groups( filtered_data = grouped_data return filtered_data + + +def _populate_instrument_header_items(met: np.ndarray) -> dict: + """ + Create header values. + + Parameters + ---------- + met : np.ndarray + Mission elapsed time. + + Returns + ------- + header : dict + Header for each instrument. + """ + sc_met = (met[0] + met[-1]) // 2 + header = {"apid": 478, + "met": sc_met, + "met_in_utc": met_to_utc(sc_met).split(".")[0], + "ttj2000ns": int(met_to_ttj2000ns(sc_met))} + return header diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 5e8c3d65bf..662900bfa1 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -13,6 +13,20 @@ def test_create_dataset(): """Tests create_dataset function.""" records = [ + { + "instrument": "mag", + "time_utc": "2025-06-20T08:00:00", + "ttj2000ns": 123456789000001, + "mag_epoch": 123456789000001, + "mag_B_GSE": [Decimal("5.0"), Decimal("-3.2"), Decimal("1.1")], + "mag_B_GSM": [Decimal("4.8"), Decimal("-3.0"), Decimal("1.0")], + "mag_B_RTN": [Decimal("5.1"), Decimal("-3.3"), Decimal("1.2")], + "mag_B_magnitude": Decimal("6.0"), + "mag_phi_B_GSM": Decimal("45.0"), + "mag_theta_B_GSM": Decimal("30.0"), + "mag_phi_B_GSE": Decimal("50.0"), + "mag_theta_B_GSE": Decimal("35.0"), + }, { "instrument": "mag", "time_utc": "2025-06-20T08:00:00", diff --git a/imap_processing/tests/ialirt/unit/test_grouping.py b/imap_processing/tests/ialirt/unit/test_grouping.py index a578713d62..aba11fac42 100644 --- a/imap_processing/tests/ialirt/unit/test_grouping.py +++ b/imap_processing/tests/ialirt/unit/test_grouping.py @@ -4,7 +4,7 @@ import pytest import xarray as xr -from imap_processing.ialirt.utils.grouping import filter_valid_groups, find_groups +from imap_processing.ialirt.utils.grouping import filter_valid_groups, find_groups, _populate_instrument_header_items @pytest.fixture @@ -96,3 +96,16 @@ def test_find_groups_no_valid(test_data): grouped_data = find_groups(nominal_data, (0, 3), "sequence", "time_seconds") assert np.all(np.unique(grouped_data["group"]) == np.array([1])) + + +def test_populate_instrument_header_items_from_met_array(): + """Test _populate_instrument_header_items.""" + + met = np.array([100.0, 101.0, 102.0, 103.0], dtype=float) + + header = _populate_instrument_header_items(met) + + expected_keys = {"apid", "met", "met_in_utc", "ttj2000ns"} + assert expected_keys.issubset(header.keys()) + + assert header["met"] == (met[0] + met[-1]) // 2 From 332aafcf7a8e6b1535079158c3cac3619b75802b Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 16 Jan 2026 13:46:50 -0700 Subject: [PATCH 42/42] test fix --- .../tests/ialirt/unit/test_create_xarray.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_create_xarray.py b/imap_processing/tests/ialirt/unit/test_create_xarray.py index 662900bfa1..5e8c3d65bf 100644 --- a/imap_processing/tests/ialirt/unit/test_create_xarray.py +++ b/imap_processing/tests/ialirt/unit/test_create_xarray.py @@ -13,20 +13,6 @@ def test_create_dataset(): """Tests create_dataset function.""" records = [ - { - "instrument": "mag", - "time_utc": "2025-06-20T08:00:00", - "ttj2000ns": 123456789000001, - "mag_epoch": 123456789000001, - "mag_B_GSE": [Decimal("5.0"), Decimal("-3.2"), Decimal("1.1")], - "mag_B_GSM": [Decimal("4.8"), Decimal("-3.0"), Decimal("1.0")], - "mag_B_RTN": [Decimal("5.1"), Decimal("-3.3"), Decimal("1.2")], - "mag_B_magnitude": Decimal("6.0"), - "mag_phi_B_GSM": Decimal("45.0"), - "mag_theta_B_GSM": Decimal("30.0"), - "mag_phi_B_GSE": Decimal("50.0"), - "mag_theta_B_GSE": Decimal("35.0"), - }, { "instrument": "mag", "time_utc": "2025-06-20T08:00:00",