From 9071fab473a2ab20f9ff742223b59eff4444ebdc Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 17 Mar 2026 09:28:51 -0700 Subject: [PATCH 1/7] Updated array serializable representation to remove values from each member. Optimize type methods for numerical types --- .../common/models/serialize/array_type.py | 80 +++++++++++++++---- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/src/fprime_gds/common/models/serialize/array_type.py b/src/fprime_gds/common/models/serialize/array_type.py index 142aff46..3542932f 100644 --- a/src/fprime_gds/common/models/serialize/array_type.py +++ b/src/fprime_gds/common/models/serialize/array_type.py @@ -4,6 +4,8 @@ @author: jishii """ +import struct + from .type_base import DictionaryType from .type_exceptions import ( ArrayLengthException, @@ -11,6 +13,7 @@ TypeMismatchException, DeserializeException, ) +from .numerical_types import NumericalType class ArrayType(DictionaryType): @@ -47,6 +50,9 @@ def validate(cls, val): for i in range(cls.LENGTH): cls.MEMBER_TYPE.validate(val[i]) + def _is_numerical_array(self) -> bool: + return issubclass(self.MEMBER_TYPE, NumericalType) + @property def val(self) -> list: """ @@ -55,7 +61,12 @@ def val(self) -> list: :return dictionary of member names to python values of member keys """ - return None if self._val is None else [item.val for item in self._val] + if self._val is None: + return None + elif self._is_numerical_array(): + return [item for item in self._val] + else: + return [item.val for item in self._val] @property def formatted_val(self) -> list: @@ -83,49 +94,86 @@ def val(self, val: list): :param val: dictionary containing python types to key names. This """ self.validate(val) - items = [self.MEMBER_TYPE(item) for item in val] + if self._is_numerical_array(): + items = [item for item in val] + else: + items = [self.MEMBER_TYPE(item) for item in val] self._val = items def to_jsonable(self): """ JSONable array object format """ + if self._is_numerical_array(): + vals = self._val + else: + vals = None if self._val is None \ + else [member.val for member in self._val] return { "name": self.__class__.__name__, "type": self.__class__.__name__, "size": self.LENGTH, "format": self.FORMAT, - "values": ( - None - if self._val is None - else [member.to_jsonable() for member in self._val] - ), + "value_type": repr(self.MEMBER_TYPE()), + "values": vals, } def serialize(self): """Serialize the array by serializing the elements one by one""" if self.val is None: raise NotInitializedException(type(self)) - return b"".join([item.serialize() for item in self._val]) + if self._is_numerical_array(): + value_format_raw = self.MEMBER_TYPE().get_serialize_format() + value_endian = '' + if self.MEMBER_TYPE.getSize() > 1: + assert value_format_raw[0] in ('>', '<'), \ + f'Expected explicit endian numerical type format but found {value_format_raw}' + value_endian = value_format_raw[0] + value_format = value_format_raw.strip('><') + + array_format = f"{value_endian}{self.LENGTH}{value_format}" + return struct.pack(array_format, *self._val) + else: + return b"".join([item.serialize() for item in self._val]) def deserialize(self, data, offset): """Deserialize the members of the array""" - values = [] - for field_index in range(self.LENGTH): + if issubclass(self.MEMBER_TYPE, NumericalType) and self.LENGTH > 0: try: - item = self.MEMBER_TYPE() - item.deserialize(data, offset) - offset += item.getSize() - values.append(item) + value_format_raw = self.MEMBER_TYPE().get_serialize_format() + value_endian = '' + if self.MEMBER_TYPE.getSize() > 1: + assert value_format_raw[0] in ('>', '<'), \ + f'Expected explicit endian numerical type format but found {value_format_raw}' + value_endian = value_format_raw[0] + value_format = value_format_raw.strip('><') + + array_format = f"{value_endian}{self.LENGTH}{value_format}" + values = struct.unpack_from(array_format, data, offset) except Exception as exc: raise DeserializeException( - f"Array index {field_index} failed to deserialize: {exc}" + f"Array NumericalType optimization failed to deserialize: {exc}" ) + else: + values = [] + for field_index in range(self.LENGTH): + try: + item = self.MEMBER_TYPE() + item.deserialize(data, offset) + offset += item.getSize() + values.append(item) + except Exception as exc: + raise DeserializeException( + f"Array index {field_index} failed to deserialize: {exc}" + ) self._val = values def getSize(self): """Return the size in bytes of the array""" - return sum(item.getSize() for item in self._val) + if self._is_numerical_array(): + return self.MEMBER_TYPE.getMaxSize() * len(self._val) + else: + return sum(item.getSize() for item in self._val) @classmethod def getMaxSize(cls): From be683206b6dc8816311cd874a765365ee6ea004d Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 17 Mar 2026 10:08:43 -0700 Subject: [PATCH 2/7] Updated Data Product decoder to decode array records using an ArrayType serializable --- src/fprime_gds/common/dp/decoder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fprime_gds/common/dp/decoder.py b/src/fprime_gds/common/dp/decoder.py index 424b8da8..d64fd1f4 100644 --- a/src/fprime_gds/common/dp/decoder.py +++ b/src/fprime_gds/common/dp/decoder.py @@ -26,6 +26,7 @@ get_dp_header_type, ) from fprime_gds.common.models.dictionaries import Dictionaries +from fprime_gds.common.models.serialize.array_type import ArrayType from fprime_gds.common.utils.config_manager import ConfigManager from fprime_gds.common.templates.dp_record_template import DpRecordTemplate @@ -183,13 +184,12 @@ def read_element(element_type): array_size_type.deserialize(array_size_data, 0) array_size = array_size_type.val - record['Size'] = array_size - record['Data'] = [] + element_array_type = ArrayType.construct_type( + record_template.get_name(), record_type, array_size, "{}" + ) - # Read each array element - for _ in range(array_size): - element_instance = read_element(record_type) - record['Data'].append(element_instance.to_jsonable()) + element_instance = read_element(element_array_type) + record['Data'] = element_instance.to_jsonable() else: # For scalar records, read the single value element_instance = read_element(record_type) From 52156b26182a51c2277b84563a76f96ba9736f0f Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 17 Mar 2026 10:34:45 -0700 Subject: [PATCH 3/7] Update gds data product unit tests to check against the new json format --- test/fprime_gds/common/dp/test_decoder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/fprime_gds/common/dp/test_decoder.py b/test/fprime_gds/common/dp/test_decoder.py index 401ac0fd..e58620e7 100644 --- a/test/fprime_gds/common/dp/test_decoder.py +++ b/test/fprime_gds/common/dp/test_decoder.py @@ -175,9 +175,9 @@ def test_decode_u8_array(self, load_dictionary, tmp_path): # Array records should have a Size field record = result["Records"][0] - assert "Size" in record assert "Data" in record - assert isinstance(record["Data"], list) + assert "size" in record["Data"] + assert isinstance(record["Data"]["values"], list) def test_decode_u32_array(self, load_dictionary, tmp_path): """Test decoding of U32 array data product.""" @@ -191,9 +191,9 @@ def test_decode_u32_array(self, load_dictionary, tmp_path): assert "Header" in result assert "Records" in result record = result["Records"][0] - assert "Size" in record assert "Data" in record - assert isinstance(record["Data"], list) + assert "size" in record["Data"] + assert isinstance(record["Data"]["values"], list) def test_decode_data_array(self, load_dictionary, tmp_path): """Test decoding of Data array data product.""" @@ -397,10 +397,10 @@ def test_decode_array_record(self, load_dictionary): assert len(records) > 0 record = records[0] - assert "Size" in record assert "Data" in record - assert isinstance(record["Data"], list) - assert len(record["Data"]) == record["Size"] + assert "size" in record["Data"] + assert isinstance(record["Data"]["values"], list) + assert len(record["Data"]["values"]) == record["Data"]["size"] class TestDataProductDecoderIntegration: From 6ea177f44c9cafb3bcb5bb8930cada94654f043d Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 17 Mar 2026 10:35:25 -0700 Subject: [PATCH 4/7] Ensure that values is a list type when using numerical optimization --- .../common/models/serialize/array_type.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/fprime_gds/common/models/serialize/array_type.py b/src/fprime_gds/common/models/serialize/array_type.py index 3542932f..d1d54f82 100644 --- a/src/fprime_gds/common/models/serialize/array_type.py +++ b/src/fprime_gds/common/models/serialize/array_type.py @@ -64,7 +64,7 @@ def val(self) -> list: if self._val is None: return None elif self._is_numerical_array(): - return [item for item in self._val] + return list(self._val) else: return [item.val for item in self._val] @@ -95,7 +95,7 @@ def val(self, val: list): """ self.validate(val) if self._is_numerical_array(): - items = [item for item in val] + items = list(val) else: items = [self.MEMBER_TYPE(item) for item in val] self._val = items @@ -104,11 +104,12 @@ def to_jsonable(self): """ JSONable array object format """ - if self._is_numerical_array(): - vals = self._val + if self._val is None: + vals = None + elif self._is_numerical_array(): + vals = list(self._val) else: - vals = None if self._val is None \ - else [member.val for member in self._val] + vals = [member.val for member in self._val] return { "name": self.__class__.__name__, "type": self.__class__.__name__, @@ -149,7 +150,7 @@ def deserialize(self, data, offset): value_format = value_format_raw.strip('><') array_format = f"{value_endian}{self.LENGTH}{value_format}" - values = struct.unpack_from(array_format, data, offset) + values = list(struct.unpack_from(array_format, data, offset)) except Exception as exc: raise DeserializeException( f"Array NumericalType optimization failed to deserialize: {exc}" From 153c962c0d94c5fa21262866fa33859f6fb0a330 Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 31 Mar 2026 14:39:14 -0700 Subject: [PATCH 5/7] Tag created ArrayTypes with record size --- src/fprime_gds/common/dp/decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fprime_gds/common/dp/decoder.py b/src/fprime_gds/common/dp/decoder.py index d64fd1f4..c76da11f 100644 --- a/src/fprime_gds/common/dp/decoder.py +++ b/src/fprime_gds/common/dp/decoder.py @@ -185,7 +185,7 @@ def read_element(element_type): array_size = array_size_type.val element_array_type = ArrayType.construct_type( - record_template.get_name(), record_type, array_size, "{}" + f'{record_template.get_name()}_{array_size}', record_type, array_size, "{}" ) element_instance = read_element(element_array_type) From cca3be7fa0e2ec625cbd7d153fccd21db4c43bac Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Tue, 31 Mar 2026 15:33:14 -0700 Subject: [PATCH 6/7] Added specialization for formatted_val --- .../common/models/serialize/array_type.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/fprime_gds/common/models/serialize/array_type.py b/src/fprime_gds/common/models/serialize/array_type.py index d1d54f82..04f8e853 100644 --- a/src/fprime_gds/common/models/serialize/array_type.py +++ b/src/fprime_gds/common/models/serialize/array_type.py @@ -77,11 +77,15 @@ def formatted_val(self) -> list: :return a formatted array """ result = [] - for item in self._val: - if hasattr(item, "formatted_val"): - result.append(item.formatted_val) - else: - result.append(self.FORMAT.format(item.val)) + if self._is_numerical_array(): + for item in self._val: + result.append(self.FORMAT.format(item)) + else: + for item in self._val: + if hasattr(item, "formatted_val"): + result.append(item.formatted_val) + else: + result.append(self.FORMAT.format(item.val)) return result @val.setter From 057ee67924fc4983dff1f4f9f811fdbf6232ff6c Mon Sep 17 00:00:00 2001 From: Gerik Kubiak Date: Thu, 9 Apr 2026 12:53:43 -0700 Subject: [PATCH 7/7] Added support for decompressing compressed data products --- src/fprime_gds/common/dp/decoder.py | 69 +++++++++++++++++---- src/fprime_gds/executables/data_products.py | 3 +- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/fprime_gds/common/dp/decoder.py b/src/fprime_gds/common/dp/decoder.py index c76da11f..4603fe6f 100644 --- a/src/fprime_gds/common/dp/decoder.py +++ b/src/fprime_gds/common/dp/decoder.py @@ -19,6 +19,9 @@ from pathlib import Path from typing import Dict, List, Any, Optional import dataclasses +import re +from io import BytesIO +import zlib from fprime_gds.common.dp.common import ( ChecksumConfig, @@ -85,7 +88,7 @@ class DataProductDecoder: - both these assumptions can be resolved by loading dictionaries (see executables/data_products.py) """ - def __init__(self, dictionaries: Dictionaries, binary_file_path: str, output_json_path: Optional[str] = None): + def __init__(self, dictionaries: Dictionaries, binary_file_path: str, disable_decompression: bool, output_json_path: Optional[str] = None): """Initialize the decoder. Args: @@ -96,6 +99,7 @@ def __init__(self, dictionaries: Dictionaries, binary_file_path: str, output_jso """ self.dictionaries = dictionaries self.binary_file_path = binary_file_path + self.disable_decompression = disable_decompression if output_json_path is None: # Generate default output path if not provided as same path with .json extension self.output_json_path = str(Path(binary_file_path).with_suffix('.json')) @@ -145,7 +149,7 @@ def decode_record(self, file_handle, record_id: int) -> Dict[str, Any]: Raises: RecordNotFoundError: If record ID not found """ - + # Query ConfigManager for record definition record_template: DpRecordTemplate = self.dictionaries.dp_record_id.get(record_id) @@ -197,6 +201,46 @@ def read_element(element_type): return record + def decode_records(self, r_io, data_size) -> List[Any]: + records = list() + position_at_start = r_io.tell() + while (r_io.tell() - position_at_start) < data_size: + # Read record ID + record_id_bin = r_io.read(ConfigManager().get_type("FwDpIdType").getSize()) + record_id_obj = ConfigManager().get_type("FwDpIdType")() + record_id_obj.deserialize(record_id_bin, 0) + record_id = record_id_obj.val + + # decode the record + record = self.decode_record(r_io, record_id) + records.append(record) + + return records + + def is_compression_record(self, record): + return re.search("dpCompressProc.CompressionRecord$", record["Record"]["record_name"]) is not None + + def decompress_records(self, records): + uncomp_bytes = bytearray() + CompressionMetadata = ConfigManager().get_type("Svc.CompressionMetadata") + + for record in records: + if not self.is_compression_record(record): + # All records must be compressed, otherwise bail + return None + + record_io = BytesIO(bytes(record["Data"]["values"])) + + record_meta = CompressionMetadata() + record_meta_data = record_io.read(record_meta.getMaxSize()) + record_meta.deserialize(record_meta_data, 0) + if record_meta.val['algorithm'] == 'UNCOMPRESSED': + uncomp_bytes.extend(record_io.read()) + elif record_meta.val['algorithm'] == 'ZLIB_DEFLATE': + uncomp_bytes.extend(zlib.decompress(record_io.read())) + + return uncomp_bytes + def decode(self) -> List[Dict[str, Any]]: """decode the entire data product file. @@ -223,16 +267,7 @@ def decode(self) -> List[Dict[str, Any]]: ##################### data_size = header_json['DataSize']["value"] position_at_start = f.tell() - while (f.tell() - position_at_start) < data_size: - # Read record ID - record_id_bin = f.read(ConfigManager().get_type("FwDpIdType").getSize()) - record_id_obj = ConfigManager().get_type("FwDpIdType")() - record_id_obj.deserialize(record_id_bin, 0) - record_id = record_id_obj.val - - # decode the record - record = self.decode_record(f, record_id) - results["Records"].append(record) + results["Records"] = self.decode_records(f, data_size) ##################### # Validate checksum # @@ -250,6 +285,16 @@ def decode(self) -> List[Dict[str, Any]]: if computed_crc != dp_crc.val: raise CRCError("Data", dp_crc.val, computed_crc) + if not self.disable_decompression and self.is_compression_record(results["Records"][0]): + + # Compressed records. Decompress and re-process + uncomp_bytes = self.decompress_records(results["Records"]) + if uncomp_bytes is not None: + uncomp_io = BytesIO(uncomp_bytes) + uncomp_records = self.decode_records(uncomp_io, len(uncomp_bytes)) + if uncomp_records is not None: + results["Records"] = uncomp_records + return results def process(self): diff --git a/src/fprime_gds/executables/data_products.py b/src/fprime_gds/executables/data_products.py index bf8eee76..b2d5b80d 100644 --- a/src/fprime_gds/executables/data_products.py +++ b/src/fprime_gds/executables/data_products.py @@ -13,6 +13,7 @@ def main(): decode_parser.add_argument("-b", "--bin-file", required=True, help="Path to input data product binary file (.fdp)") decode_parser.add_argument("-d", "--dictionary", required=True, help="Path to F Prime JSON Dictionary") decode_parser.add_argument("-o", "--output", required=False, help="Path to output JSON file (defaults to .json)") + decode_parser.add_argument("-z", "--disable-decompression", action='store_true', help="Disable automatic decompression of data products") validate_parser = subcommands_parser.add_parser('validate', help='Validate a data product') validate_parser.add_argument("-b", "--bin-file", required=True, help="Path to input data product binary file (.fdp)") @@ -29,7 +30,7 @@ def main(): if args.command == "decode": assert args.dictionaries is not None, "Dictionaries must be loaded" - DataProductDecoder(args.dictionaries, args.bin_file, args.output).process() + DataProductDecoder(args.dictionaries, args.bin_file, args.disable_decompression, args.output).process() elif args.command == "validate": success = DataProductValidator(