From 835ab0fc68f8eb639e0a242256b9875d8f03bac6 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Mon, 13 Jul 2026 17:31:49 +0200 Subject: [PATCH] Apply J1939 metadata parsing also on proprietary frames - Exclude non-J1939 frames (such as 11 bit IDs) by writing null on their metadata struct --- decoder.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/decoder.py b/decoder.py index 334a070..c689a74 100644 --- a/decoder.py +++ b/decoder.py @@ -3,14 +3,12 @@ import polars as pl def get_j1939_mask() -> pl.Expr: """ - Returns a Polars expression representing the strict J1939 filtering rules. + Returns a Polars expression representing the J1939 filtering rules. + Updated to include all 29-bit extended frames, including proprietary ones + (which have the Data Page bit set to 1) and CAN FD frames. """ id_int = pl.col("ID").str.to_integer(base=16).cast(pl.UInt32) - return ( - (id_int > 0x7FF) & - ((id_int % 33554432 // 16777216) == 0) & - (pl.col("DLC") <= 8) - ) + return id_int > 0x7FF def decode_j1939_metadata(lf: pl.LazyFrame) -> pl.LazyFrame: """ @@ -44,7 +42,7 @@ def decode_j1939_metadata(lf: pl.LazyFrame) -> pl.LazyFrame: def decode_j1939_frames(df: pl.DataFrame) -> pl.DataFrame: id_int = pl.col("ID").str.to_integer(base=16).cast(pl.UInt32) - is_j1939 = (id_int > 0x7FF) & ((id_int % 33554432 // 16777216) == 0) & (pl.col("DLC") <= 8) + is_j1939 = id_int > 0x7FF priority = (id_int // 67108864) % 8 pf = (id_int // 65536) % 256 @@ -64,10 +62,13 @@ def decode_j1939_frames(df: pl.DataFrame) -> pl.DataFrame: ]) ).otherwise(None) + return df.with_columns(j1939_meta.alias("j1939_metadata")) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="J1939 decoder") parser.add_argument("input_parquet", help="Path to the raw .parquet file") - parser.add_argument("output_parquet", help="Path to save the decoded .parquet file") + parser.add_argument("output_paquet", help="Path to save the decoded .parquet file") args = parser.parse_args() df = pl.scan_parquet(args.input_parquet).collect()