diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py
index feac832fdf..20bb16ee4b 100644
--- a/imap_processing/glows/l1a/glows_l1a.py
+++ b/imap_processing/glows/l1a/glows_l1a.py
@@ -9,6 +9,7 @@
from imap_processing.glows.l0.decom_glows import decom_packets
from imap_processing.glows.l0.glows_l0_data import DirectEventL0
from imap_processing.glows.l1a.glows_l1a_data import DirectEventL1A, HistogramL1A
+from imap_processing.glows.utils.constants import GlowsConstants
from imap_processing.spice.time import (
met_to_ttj2000ns,
)
@@ -127,6 +128,9 @@ def generate_de_dataset(
"""
# TODO: Block header per second, or global attribute?
+ # Filter out DE records with no direct_events (incomplete packet sequences)
+ de_l1a_list = [de for de in de_l1a_list if de.direct_events is not None]
+
# Store timestamps for each DirectEventL1a object.
time_data = np.zeros(len(de_l1a_list), dtype=np.int64)
@@ -292,10 +296,13 @@ def generate_histogram_dataset(
"""
# Store timestamps for each HistogramL1A object.
time_data = np.zeros(len(hist_l1a_list), dtype=np.int64)
- # TODO Add daily average of histogram counts
# Data in lists, for each of the 25 time varying datapoints in HistogramL1A
- hist_data = np.zeros((len(hist_l1a_list), 3600), dtype=np.uint16)
+ hist_data = np.full(
+ (len(hist_l1a_list), GlowsConstants.STANDARD_BIN_COUNT),
+ GlowsConstants.HISTOGRAM_FILLVAL,
+ dtype=np.uint16,
+ )
# First variable is the output data type, second is the list of values
support_data: dict = {
@@ -326,7 +333,9 @@ def generate_histogram_dataset(
for index, hist in enumerate(hist_l1a_list):
epoch_time = met_to_ttj2000ns(hist.imap_start_time.to_seconds())
- hist_data[index] = hist.histogram
+ # Assign histogram data, padding with zeros if shorter than max_bins
+ hist_len = len(hist.histogram)
+ hist_data[index, :hist_len] = hist.histogram
support_data["flags_set_onboard"][1].append(hist.flags["flags_set_onboard"])
support_data["is_generated_on_ground"][1].append(
@@ -348,7 +357,8 @@ def generate_histogram_dataset(
dims=["epoch"],
attrs=glows_cdf_attributes.get_variable_attributes("epoch", check_schema=False),
)
- bin_count = 3600 # TODO: Is it always 3600 bins?
+
+ bin_count = GlowsConstants.STANDARD_BIN_COUNT
bins = xr.DataArray(
np.arange(bin_count),
diff --git a/imap_processing/glows/l1a/glows_l1a_data.py b/imap_processing/glows/l1a/glows_l1a_data.py
index 0ebad91c5c..550e6681e9 100644
--- a/imap_processing/glows/l1a/glows_l1a_data.py
+++ b/imap_processing/glows/l1a/glows_l1a_data.py
@@ -1,5 +1,6 @@
"""Data classes to support GLOWS L1A processing."""
+import logging
import struct
from dataclasses import InitVar, dataclass, field
@@ -7,6 +8,8 @@
from imap_processing.glows.l0.glows_l0_data import DirectEventL0, HistogramL0
from imap_processing.glows.utils.constants import DirectEvent, TimeTuple
+logger = logging.getLogger(__name__)
+
@dataclass
class StatusData:
@@ -261,8 +264,7 @@ def __post_init__(self, l0: HistogramL0) -> None:
self.glows_time_offset = TimeTuple(l0.GLXOFFSEC, l0.GLXOFFSUBSEC)
# In L1a, these are left as unit encoded values.
- # TODO: This is plus one in validation code, why?
- self.number_of_spins_per_block = l0.SPINS + 1
+ self.number_of_spins_per_block = l0.SPINS
self.number_of_bins_per_histogram = l0.NBINS
self.number_of_events = l0.EVENTS
self.filter_temperature_average = l0.TEMPAVG
@@ -280,6 +282,16 @@ def __post_init__(self, l0: HistogramL0) -> None:
"is_generated_on_ground": False,
}
+ # Remove the extra byte from some packets (if there are an odd number of bins)
+ if self.number_of_bins_per_histogram % 2 == 1:
+ self.histogram = self.histogram[:-1]
+
+ if self.number_of_bins_per_histogram != len(self.histogram):
+ logger.warning(
+ f"Number of bins {self.number_of_bins_per_histogram} does not match "
+ f"processed number of bins {len(self.histogram)}!"
+ )
+
@dataclass
class DirectEventL1A:
diff --git a/imap_processing/glows/packet_definitions/GLX_COMBINED.xml b/imap_processing/glows/packet_definitions/GLX_COMBINED.xml
index ebca7a4e48..e86bc8e229 100644
--- a/imap_processing/glows/packet_definitions/GLX_COMBINED.xml
+++ b/imap_processing/glows/packet_definitions/GLX_COMBINED.xml
@@ -35,10 +35,14 @@ P_GLX_TMSCDE.xml and P_GLX_TMSHIST.xml to process both packet APIDs -->
-
+
- 28800
+
+
+
+
+
@@ -172,9 +176,9 @@ P_GLX_TMSCDE.xml and P_GLX_TMSHIST.xml to process both packet APIDs -->
Number of events
Number of event in all bins (sum) of this histogram.
-
+
Histogram Counts
- Total histogram data counts. Each bin has 8 bits of data, with 3600 total bins.
+ Total histogram data counts. Each bin has 8 bits of data. Number of bins is dynamically determined from packet length.
diff --git a/imap_processing/glows/utils/constants.py b/imap_processing/glows/utils/constants.py
index 357535ace7..13821f70ae 100644
--- a/imap_processing/glows/utils/constants.py
+++ b/imap_processing/glows/utils/constants.py
@@ -61,10 +61,16 @@ class GlowsConstants:
IMAP clock)
SCAN_CIRCLE_ANGULAR_RADIUS: float
angular radius of IMAP/GLOWS scanning circle [deg]
+ HISTOGRAM_FILLVAL: int
+ Fill value for histogram bins (65535 for uint16)
+ STANDARD_BIN_COUNT: int
+ Standard number of bins per histogram (3600)
"""
SUBSECOND_LIMIT: int = 2_000_000
SCAN_CIRCLE_ANGULAR_RADIUS: float = 75.0
+ HISTOGRAM_FILLVAL: int = 65535
+ STANDARD_BIN_COUNT: int = 3600
@dataclass
diff --git a/imap_processing/tests/glows/test_glows_l1a_data.py b/imap_processing/tests/glows/test_glows_l1a_data.py
index ec7c422bcc..9877ddc1be 100644
--- a/imap_processing/tests/glows/test_glows_l1a_data.py
+++ b/imap_processing/tests/glows/test_glows_l1a_data.py
@@ -41,6 +41,45 @@ def test_histogram_list(histogram_test_data, decom_test_data):
assert sum(histogram_test_data.histogram) == histl0.EVENTS
+def test_histogram_bin_handling(decom_test_data):
+ """Test histogram bin handling for odd bins."""
+ histl0 = decom_test_data[0][0]
+
+ # Test odd number of bins - extra byte should be removed
+ mock_histl0_odd = mock.MagicMock(spec=histl0)
+ mock_histl0_odd.NBINS = 3599
+ mock_histl0_odd.HISTOGRAM_DATA = list(range(3600))
+ mock_histl0_odd.SWVER = 1
+ mock_histl0_odd.packet_file_name = "test.pkts"
+ mock_histl0_odd.ccsds_header = mock.MagicMock()
+ mock_histl0_odd.ccsds_header.SRC_SEQ_CTR = 0
+ mock_histl0_odd.STARTID = 0
+ mock_histl0_odd.ENDID = 10
+ mock_histl0_odd.SEC = 1000
+ mock_histl0_odd.SUBSEC = 0
+ mock_histl0_odd.OFFSETSEC = 0
+ mock_histl0_odd.OFFSETSUBSEC = 0
+ mock_histl0_odd.GLXSEC = 1000
+ mock_histl0_odd.GLXSUBSEC = 0
+ mock_histl0_odd.GLXOFFSEC = 0
+ mock_histl0_odd.GLXOFFSUBSEC = 0
+ mock_histl0_odd.SPINS = 10
+ mock_histl0_odd.EVENTS = 1000
+ mock_histl0_odd.TEMPAVG = 100
+ mock_histl0_odd.TEMPVAR = 10
+ mock_histl0_odd.HVAVG = 500
+ mock_histl0_odd.HVVAR = 5
+ mock_histl0_odd.SPAVG = 150
+ mock_histl0_odd.SPVAR = 1
+ mock_histl0_odd.ELAVG = 20
+ mock_histl0_odd.ELVAR = 2
+ mock_histl0_odd.FLAGS = 0
+
+ hist_odd = HistogramL1A(mock_histl0_odd)
+ assert len(hist_odd.histogram) == 3599
+ assert hist_odd.number_of_bins_per_histogram == 3599
+
+
def test_histogram_obs_day(packet_path):
l1a = glows_l1a(packet_path)
@@ -522,10 +561,11 @@ def test_expected_hist_results(l1a_dataset):
}
# block header and flags are handled differently, so not tested here
+ # "number_of_spins_per_block" is a special case and handled specifically
+ # (validation data is incorrect)
compare_fields = [
"first_spin_id",
"last_spin_id",
- "number_of_spins_per_block",
"number_of_bins_per_histogram",
"histogram",
"number_of_events",
@@ -560,6 +600,10 @@ def test_expected_hist_results(l1a_dataset):
for field in compare_fields:
assert np.array_equal(data[field], datapoint[field].data)
+ assert np.array_equal(
+ data["number_of_spins_per_block"] - 1,
+ datapoint["number_of_spins_per_block"].data,
+ )
@mock.patch("imap_processing.glows.l1a.glows_l1a.decom_packets")
diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py
index cac1d7c0c2..b2595f3c0b 100644
--- a/imap_processing/tests/glows/test_glows_l1b_data.py
+++ b/imap_processing/tests/glows/test_glows_l1b_data.py
@@ -111,7 +111,6 @@ def test_validation_data_histogram(
"glows_end_time_offset": "glows_time_offset",
"imap_start_time": "imap_start_time",
"imap_end_time_offset": "imap_time_offset",
- "number_of_spins_per_block": "number_of_spins_per_block",
"number_of_bins_per_histogram": "number_of_bins_per_histogram",
"histogram": "histogram",
"number_of_events": "number_of_events",