From 6891852a6c2b4405399e10240b1703f875fd8acb Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Tue, 18 Nov 2025 10:48:36 -0700 Subject: [PATCH 1/6] Update equation for ena_intensity_sys_err --- imap_processing/hi/hi_l2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index e090c4a843..23d86ccb93 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -305,7 +305,11 @@ def calculate_ena_intensity( map_ds["ena_intensity_stat_uncert"] = ( map_ds["ena_signal_rate_stat_unc"] / flux_conversion_divisor ) - map_ds["ena_intensity_sys_err"] = map_ds["bg_rates_unc"] / flux_conversion_divisor + map_ds["ena_intensity_sys_err"] = ( + np.sqrt(map_ds["bg_rates_unc"] / map_ds["exposure_factor"]) + / map_ds["exposure_factor"] + / flux_conversion_divisor + ) # Combine calibration products using proper weighted averaging # as described in Hi Algorithm Document Section 3.1.2 From 3e26124b654825a7c9c3f26e2909ef420cb7e950 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Thu, 4 Dec 2025 12:12:18 -0700 Subject: [PATCH 2/6] Fix systematic error calculation - multiply instead of divide --- imap_processing/hi/hi_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index 23d86ccb93..74f11eee0d 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -306,7 +306,7 @@ def calculate_ena_intensity( map_ds["ena_signal_rate_stat_unc"] / flux_conversion_divisor ) map_ds["ena_intensity_sys_err"] = ( - np.sqrt(map_ds["bg_rates_unc"] / map_ds["exposure_factor"]) + np.sqrt(map_ds["bg_rates_unc"] * map_ds["exposure_factor"]) / map_ds["exposure_factor"] / flux_conversion_divisor ) From 0baf9d6571ee35dcb903e64ead792336000357c0 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Mon, 8 Dec 2025 09:07:10 -0700 Subject: [PATCH 3/6] Fix systematic error calculation - use bg_rates, not bg_rates_unc --- imap_processing/hi/hi_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index 74f11eee0d..8b1a898ffa 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -306,7 +306,7 @@ def calculate_ena_intensity( map_ds["ena_signal_rate_stat_unc"] / flux_conversion_divisor ) map_ds["ena_intensity_sys_err"] = ( - np.sqrt(map_ds["bg_rates_unc"] * map_ds["exposure_factor"]) + np.sqrt(map_ds["bg_rates"] * map_ds["exposure_factor"]) / map_ds["exposure_factor"] / flux_conversion_divisor ) From 1507a6803be8912c2e3690a137eb7a801ded9ff0 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Wed, 10 Dec 2025 08:06:04 -0700 Subject: [PATCH 4/6] Remove systematic error from weights used to combine calibration products --- imap_processing/hi/hi_l2.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index 8b1a898ffa..c4cd6a48c8 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -371,16 +371,11 @@ def combine_calibration_products( map_ds, geometric_factors, esa_energies ) - # Calculate total variance - # Note that sys_err contains uncertainty, so it must be squared to get - # the systematic variance needed in this equation. - total_variance = improved_stat_variance + sys_err**2 - # Perform inverse-variance weighted averaging # Handle divide by zero and invalid values with np.errstate(divide="ignore", invalid="ignore"): # Use total variance weights for flux combination - flux_weights = 1.0 / total_variance + flux_weights = 1.0 / improved_stat_variance weighted_flux_sum = (ena_flux * flux_weights).sum(dim="calibration_prod") combined_flux = weighted_flux_sum / flux_weights.sum(dim="calibration_prod") From 84537f04f981af801d1b9ca4e0556e9a8f905f62 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Wed, 10 Dec 2025 08:09:20 -0700 Subject: [PATCH 5/6] Update test to remove use of statistical uncertainty when combinging cal products --- imap_processing/tests/hi/test_hi_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/tests/hi/test_hi_l2.py b/imap_processing/tests/hi/test_hi_l2.py index 215ca2bfa0..1ada7f6f1b 100644 --- a/imap_processing/tests/hi/test_hi_l2.py +++ b/imap_processing/tests/hi/test_hi_l2.py @@ -644,7 +644,7 @@ def test_statistical_uncertainty_combination_correctness(): # Manual calculation of expected statistical uncertainty combination # combined_stat_unc = sqrt(1/sum(1 / stat_unc**2)) expected_combined_stat_unc = np.sqrt(1 / np.sum(1 / stat_unc_values**2)) - flux_weights = 1.0 / (np.array([101, 101]) + np.array([4, 16])) + flux_weights = 1.0 / np.array([101, 101]) expected_flux = np.sum(flux_values.squeeze() * flux_weights) / np.sum(flux_weights) np.testing.assert_almost_equal( From 1f9faf464a48d805ef88d4b775e758d8b51a18b8 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Mon, 15 Dec 2025 10:17:27 -0700 Subject: [PATCH 6/6] Get rid of check that variance is >= 1 in calculation of imporved variance --- imap_processing/hi/hi_l2.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index c4cd6a48c8..08c250129e 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -449,11 +449,6 @@ def _calculate_improved_stat_variance( # Total count rates for Poisson uncertainty calculation total_count_rates_for_uncertainty = map_ds["bg_rates"] + averaged_signal_rates - # Ensure non-negative values for sqrt and minimum of 1 for uncertainty calculation - total_count_rates_for_uncertainty = xr.where( - total_count_rates_for_uncertainty < 1, 1, total_count_rates_for_uncertainty - ) - logger.debug("Computing improved flux uncertainties") # Statistical variance: with np.errstate(divide="ignore", invalid="ignore"):