GLOWS l1a validation corrections

imap_processing/glows/l1a/glows_l1a.py

@@ -9,6 +9,7 @@
 from imap_processing.glows.l0.decom_glows import decom_packets
 from imap_processing.glows.l0.glows_l0_data import DirectEventL0
 from imap_processing.glows.l1a.glows_l1a_data import DirectEventL1A, HistogramL1A
+from imap_processing.glows.utils.constants import GlowsConstants
 from imap_processing.spice.time import (
     met_to_ttj2000ns,
 )
@@ -127,6 +128,9 @@ def generate_de_dataset(
     """
     # TODO: Block header per second, or global attribute?
 
+    # Filter out DE records with no direct_events (incomplete packet sequences)
+    de_l1a_list = [de for de in de_l1a_list if de.direct_events is not None]
+
     # Store timestamps for each DirectEventL1a object.
     time_data = np.zeros(len(de_l1a_list), dtype=np.int64)
 
@@ -292,10 +296,13 @@ def generate_histogram_dataset(
     """
     # Store timestamps for each HistogramL1A object.
     time_data = np.zeros(len(hist_l1a_list), dtype=np.int64)
-    # TODO Add daily average of histogram counts
     # Data in lists, for each of the 25 time varying datapoints in HistogramL1A
 
-    hist_data = np.zeros((len(hist_l1a_list), 3600), dtype=np.uint16)
+    hist_data = np.full(
+        (len(hist_l1a_list), GlowsConstants.STANDARD_BIN_COUNT),
+        GlowsConstants.HISTOGRAM_FILLVAL,
+        dtype=np.uint16,
+    )
 
     # First variable is the output data type, second is the list of values
     support_data: dict = {
@@ -326,7 +333,9 @@ def generate_histogram_dataset(
 
     for index, hist in enumerate(hist_l1a_list):
         epoch_time = met_to_ttj2000ns(hist.imap_start_time.to_seconds())
-        hist_data[index] = hist.histogram
+        # Assign histogram data, padding with zeros if shorter than max_bins
+        hist_len = len(hist.histogram)
+        hist_data[index, :hist_len] = hist.histogram
 
         support_data["flags_set_onboard"][1].append(hist.flags["flags_set_onboard"])
         support_data["is_generated_on_ground"][1].append(
@@ -348,7 +357,8 @@ def generate_histogram_dataset(
         dims=["epoch"],
         attrs=glows_cdf_attributes.get_variable_attributes("epoch", check_schema=False),
     )
-    bin_count = 3600  # TODO: Is it always 3600 bins?
+
+    bin_count = GlowsConstants.STANDARD_BIN_COUNT
 
     bins = xr.DataArray(
         np.arange(bin_count),

imap_processing/glows/l1a/glows_l1a_data.py

@@ -1,12 +1,15 @@
 """Data classes to support GLOWS L1A processing."""
 
+import logging
 import struct
 from dataclasses import InitVar, dataclass, field
 
 from imap_processing.glows import __version__
 from imap_processing.glows.l0.glows_l0_data import DirectEventL0, HistogramL0
 from imap_processing.glows.utils.constants import DirectEvent, TimeTuple
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class StatusData:
@@ -261,8 +264,7 @@ def __post_init__(self, l0: HistogramL0) -> None:
         self.glows_time_offset = TimeTuple(l0.GLXOFFSEC, l0.GLXOFFSUBSEC)
 
         # In L1a, these are left as unit encoded values.
-        # TODO: This is plus one in validation code, why?
-        self.number_of_spins_per_block = l0.SPINS + 1
+        self.number_of_spins_per_block = l0.SPINS
         self.number_of_bins_per_histogram = l0.NBINS
         self.number_of_events = l0.EVENTS
         self.filter_temperature_average = l0.TEMPAVG
@@ -280,6 +282,16 @@ def __post_init__(self, l0: HistogramL0) -> None:
             "is_generated_on_ground": False,
         }
 
+        # Remove the extra byte from some packets (if there are an odd number of bins)
+        if self.number_of_bins_per_histogram % 2 == 1:
+            self.histogram = self.histogram[:-1]
+
+        if self.number_of_bins_per_histogram != len(self.histogram):
+            logger.warning(
+                f"Number of bins {self.number_of_bins_per_histogram} does not match "
+                f"processed number of bins {len(self.histogram)}!"
+            )
+
 
 @dataclass
 class DirectEventL1A:

imap_processing/glows/packet_definitions/GLX_COMBINED.xml

@@ -35,10 +35,14 @@ P_GLX_TMSCDE.xml and P_GLX_TMSHIST.xml to process both packet APIDs -->
 			<xtce:IntegerParameterType name="INT32" signed="true">
 				<xtce:IntegerDataEncoding sizeInBits="32" encoding="signed" />
 			</xtce:IntegerParameterType>
-			<xtce:BinaryParameterType name="BYTE28800">
+			<xtce:BinaryParameterType name="BYTEHIST">
 				<xtce:BinaryDataEncoding bitOrder="mostSignificantBitFirst">
 					<xtce:SizeInBits>
-						<xtce:FixedValue>28800</xtce:FixedValue>
+						<xtce:DynamicValue>
+						  <xtce:ParameterInstanceRef parameterRef="PKT_LEN"/>
+<!--To find the rest of the packet, this intercept is -(32 + 32 + 16 + 16 + 24 + 32 + 24 + 16 + 24 + 32 + 24 + 16 + 24 + 8 + 16 + 8 + 16 + 16 + 32 + 16 + 32 + 8 + 16 + 32) = 512 bits for all HIST fields before HISTOGRAM_DATA. Then +8 because PKT_LEN is zero indexed, resulting in an offset of -504. -->
+						  <xtce:LinearAdjustment intercept="-504" slope="8"/>
+						</xtce:DynamicValue>
 					</xtce:SizeInBits>
 				</xtce:BinaryDataEncoding>
 			</xtce:BinaryParameterType>
@@ -172,9 +176,9 @@ P_GLX_TMSCDE.xml and P_GLX_TMSHIST.xml to process both packet APIDs -->
 				<xtce:ShortDescription>Number of events</xtce:ShortDescription>
 				<xtce:LongDescription>Number of event in all bins (sum) of this histogram.</xtce:LongDescription>
 			</xtce:Parameter>
-			<xtce:Parameter name="HISTOGRAM_DATA" parameterTypeRef="BYTE28800">
+			<xtce:Parameter name="HISTOGRAM_DATA" parameterTypeRef="BYTEHIST">
 				<xtce:ShortDescription>Histogram Counts</xtce:ShortDescription>
-				<xtce:LongDescription>Total histogram data counts. Each bin has 8 bits of data, with 3600 total bins.</xtce:LongDescription>
+				<xtce:LongDescription>Total histogram data counts. Each bin has 8 bits of data. Number of bins is dynamically determined from packet length.</xtce:LongDescription>
 			</xtce:Parameter>
 			<!-- Direct event parameters -->
 			<xtce:Parameter name="LEN" parameterTypeRef="UINT16">

imap_processing/glows/utils/constants.py

@@ -61,10 +61,16 @@ class GlowsConstants:
         IMAP clock)
     SCAN_CIRCLE_ANGULAR_RADIUS: float
         angular radius of IMAP/GLOWS scanning circle [deg]
+    HISTOGRAM_FILLVAL: int
+        Fill value for histogram bins (65535 for uint16)
+    STANDARD_BIN_COUNT: int
+        Standard number of bins per histogram (3600)
     """
 
     SUBSECOND_LIMIT: int = 2_000_000
     SCAN_CIRCLE_ANGULAR_RADIUS: float = 75.0
+    HISTOGRAM_FILLVAL: int = 65535
+    STANDARD_BIN_COUNT: int = 3600
 
 
 @dataclass

imap_processing/tests/glows/test_glows_l1a_data.py

@@ -41,6 +41,45 @@ def test_histogram_list(histogram_test_data, decom_test_data):
     assert sum(histogram_test_data.histogram) == histl0.EVENTS
 
 
+def test_histogram_bin_handling(decom_test_data):
+    """Test histogram bin handling for odd bins."""
+    histl0 = decom_test_data[0][0]
+
+    # Test odd number of bins - extra byte should be removed
+    mock_histl0_odd = mock.MagicMock(spec=histl0)
+    mock_histl0_odd.NBINS = 3599
+    mock_histl0_odd.HISTOGRAM_DATA = list(range(3600))
+    mock_histl0_odd.SWVER = 1
+    mock_histl0_odd.packet_file_name = "test.pkts"
+    mock_histl0_odd.ccsds_header = mock.MagicMock()
+    mock_histl0_odd.ccsds_header.SRC_SEQ_CTR = 0
+    mock_histl0_odd.STARTID = 0
+    mock_histl0_odd.ENDID = 10
+    mock_histl0_odd.SEC = 1000
+    mock_histl0_odd.SUBSEC = 0
+    mock_histl0_odd.OFFSETSEC = 0
+    mock_histl0_odd.OFFSETSUBSEC = 0
+    mock_histl0_odd.GLXSEC = 1000
+    mock_histl0_odd.GLXSUBSEC = 0
+    mock_histl0_odd.GLXOFFSEC = 0
+    mock_histl0_odd.GLXOFFSUBSEC = 0
+    mock_histl0_odd.SPINS = 10
+    mock_histl0_odd.EVENTS = 1000
+    mock_histl0_odd.TEMPAVG = 100
+    mock_histl0_odd.TEMPVAR = 10
+    mock_histl0_odd.HVAVG = 500
+    mock_histl0_odd.HVVAR = 5
+    mock_histl0_odd.SPAVG = 150
+    mock_histl0_odd.SPVAR = 1
+    mock_histl0_odd.ELAVG = 20
+    mock_histl0_odd.ELVAR = 2
+    mock_histl0_odd.FLAGS = 0
+
+    hist_odd = HistogramL1A(mock_histl0_odd)
+    assert len(hist_odd.histogram) == 3599
+    assert hist_odd.number_of_bins_per_histogram == 3599
+
+
 def test_histogram_obs_day(packet_path):
     l1a = glows_l1a(packet_path)
 
@@ -522,10 +561,11 @@ def test_expected_hist_results(l1a_dataset):
     }
 
     # block header and flags are handled differently, so not tested here
+    # "number_of_spins_per_block" is a special case and handled specifically
+    # (validation data is incorrect)
     compare_fields = [
         "first_spin_id",
         "last_spin_id",
-        "number_of_spins_per_block",
         "number_of_bins_per_histogram",
         "histogram",
         "number_of_events",
@@ -560,6 +600,10 @@ def test_expected_hist_results(l1a_dataset):
 
         for field in compare_fields:
             assert np.array_equal(data[field], datapoint[field].data)
+        assert np.array_equal(
+            data["number_of_spins_per_block"] - 1,
+            datapoint["number_of_spins_per_block"].data,
+        )
 
 
 @mock.patch("imap_processing.glows.l1a.glows_l1a.decom_packets")

imap_processing/tests/glows/test_glows_l1b_data.py

@@ -111,7 +111,6 @@ def test_validation_data_histogram(
         "glows_end_time_offset": "glows_time_offset",
         "imap_start_time": "imap_start_time",
         "imap_end_time_offset": "imap_time_offset",
-        "number_of_spins_per_block": "number_of_spins_per_block",
         "number_of_bins_per_histogram": "number_of_bins_per_histogram",
         "histogram": "histogram",
         "number_of_events": "number_of_events",