From 21172e020b209c5e176c9aba896d3d8e1fc8c973 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Thu, 9 Jul 2026 23:39:25 +0200 Subject: [PATCH] Keep parquet data in hexadecimal format --- src/analyzer.py | 0 src/parser.py | 9 ++++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 src/analyzer.py diff --git a/src/analyzer.py b/src/analyzer.py new file mode 100644 index 0000000..e69de29 diff --git a/src/parser.py b/src/parser.py index a4e76d9..5b7b634 100644 --- a/src/parser.py +++ b/src/parser.py @@ -63,7 +63,7 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N def parse_csv(csv_path: PathLike) -> pl.LazyFrame: """ - Ingests a parsed CSV file, unpacks hex strings into 8 integer columns, + Ingests a parsed CSV file, unpacks hex strings into 8 hex columns, generates sequential timestamps if missing, and returns a Polars LazyFrame. """ lf = pl.scan_csv(csv_path, schema_overrides={"ID": pl.String, "Data": pl.String}) @@ -76,12 +76,11 @@ def parse_csv(csv_path: PathLike) -> pl.LazyFrame: expr = ( pl.col("Data").str.strip_chars().str.split(" ") .list.get(i, null_on_oob=True) - .str.to_integer(base=16, strict=False) - .alias(f"b{i}").cast(pl.UInt8) + .alias(f"b{i}") ) byte_exprs.append(expr) - lf = lf.with_columns(byte_exprs).drop("Data") + lf = lf.with_columns(byte_exprs) return lf.with_columns([ pl.col("DLC").cast(pl.UInt8), @@ -112,5 +111,5 @@ if __name__ == '__main__': elif args.command == "parquet": print(f"[*] Processing {args.input_csv}...") lf = parse_csv(args.input_csv) - save_parquet(lf, args.output_parquet) + lf.sink_parquet(args.output_parquet) print(f"[+] Saved parquet file to {args.output_parquet}")