Keep parquet data in hexadecimal format
This commit is contained in:
+4
-5
@@ -63,7 +63,7 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
|
||||
|
||||
def parse_csv(csv_path: PathLike) -> pl.LazyFrame:
|
||||
"""
|
||||
Ingests a parsed CSV file, unpacks hex strings into 8 integer columns,
|
||||
Ingests a parsed CSV file, unpacks hex strings into 8 hex columns,
|
||||
generates sequential timestamps if missing, and returns a Polars LazyFrame.
|
||||
"""
|
||||
lf = pl.scan_csv(csv_path, schema_overrides={"ID": pl.String, "Data": pl.String})
|
||||
@@ -76,12 +76,11 @@ def parse_csv(csv_path: PathLike) -> pl.LazyFrame:
|
||||
expr = (
|
||||
pl.col("Data").str.strip_chars().str.split(" ")
|
||||
.list.get(i, null_on_oob=True)
|
||||
.str.to_integer(base=16, strict=False)
|
||||
.alias(f"b{i}").cast(pl.UInt8)
|
||||
.alias(f"b{i}")
|
||||
)
|
||||
byte_exprs.append(expr)
|
||||
|
||||
lf = lf.with_columns(byte_exprs).drop("Data")
|
||||
lf = lf.with_columns(byte_exprs)
|
||||
|
||||
return lf.with_columns([
|
||||
pl.col("DLC").cast(pl.UInt8),
|
||||
@@ -112,5 +111,5 @@ if __name__ == '__main__':
|
||||
elif args.command == "parquet":
|
||||
print(f"[*] Processing {args.input_csv}...")
|
||||
lf = parse_csv(args.input_csv)
|
||||
save_parquet(lf, args.output_parquet)
|
||||
lf.sink_parquet(args.output_parquet)
|
||||
print(f"[+] Saved parquet file to {args.output_parquet}")
|
||||
|
||||
Reference in New Issue
Block a user