eeeck/CANveyor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 6 Wiki Activity

Keep parquet data in hexadecimal format

Browse Source
This commit is contained in:
Erick
2026-07-09 23:39:25 +02:00
parent ce20587591
commit 21172e020b
2 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/analyzer.py
View File
src/parser.py
+4 -5
View File
@@ -63,7 +63,7 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
def parse_csv(csv_path: PathLike) -> pl.LazyFrame: def parse_csv(csv_path: PathLike) -> pl.LazyFrame:
""" """
Ingests a parsed CSV file, unpacks hex strings into 8 integer columns, Ingests a parsed CSV file, unpacks hex strings into 8 hex columns,
generates sequential timestamps if missing, and returns a Polars LazyFrame. generates sequential timestamps if missing, and returns a Polars LazyFrame.
""" """
lf = pl.scan_csv(csv_path, schema_overrides={"ID": pl.String, "Data": pl.String}) lf = pl.scan_csv(csv_path, schema_overrides={"ID": pl.String, "Data": pl.String})
@@ -76,12 +76,11 @@ def parse_csv(csv_path: PathLike) -> pl.LazyFrame:
expr = ( expr = (
pl.col("Data").str.strip_chars().str.split(" ") pl.col("Data").str.strip_chars().str.split(" ")
.list.get(i, null_on_oob=True) .list.get(i, null_on_oob=True)
.str.to_integer(base=16, strict=False) .alias(f"b{i}")
.alias(f"b{i}").cast(pl.UInt8)
) )
byte_exprs.append(expr) byte_exprs.append(expr)
lf = lf.with_columns(byte_exprs).drop("Data") lf = lf.with_columns(byte_exprs)
return lf.with_columns([ return lf.with_columns([
pl.col("DLC").cast(pl.UInt8), pl.col("DLC").cast(pl.UInt8),
@@ -112,5 +111,5 @@ if __name__ == '__main__':
elif args.command == "parquet": elif args.command == "parquet":
print(f"[*] Processing {args.input_csv}...") print(f"[*] Processing {args.input_csv}...")
lf = parse_csv(args.input_csv) lf = parse_csv(args.input_csv)
save_parquet(lf, args.output_parquet) lf.sink_parquet(args.output_parquet)
print(f"[+] Saved parquet file to {args.output_parquet}") print(f"[+] Saved parquet file to {args.output_parquet}")