Refactor statistical analysis modules for performance

- Optimize data processing pipelines across files by replacing iterative
  pandas operations with vectorized NumPy routines

stat/utils/extractor.py

@@ -7,9 +7,9 @@ from pathlib import Path
 
 import numpy as np
 import pandas as pd
+import polars as pl
 
 def to_int(x):
-    """Convert a hex string or integer to int, returning NaN on failure."""
     if isinstance(x, (int, np.integer)):
         return int(x)
     if isinstance(x, str):
@@ -20,7 +20,6 @@ def to_int(x):
     return np.nan
 
 def extract_id(row: pd.Series) -> str:
-    """Extracts PGN from metadata or falls back to CAN ID."""
     meta = row.get('j1939_metadata')
     if pd.isna(meta):
         return f"ID: {row['ID']}"
@@ -34,7 +33,31 @@ def extract_id(row: pd.Series) -> str:
     return f"ID: {row['ID']}"
 
 def load_data(file_path: Path) -> pd.DataFrame:
-    """Loads Parquet file and adds an Identifier column."""
-    df = pd.read_parquet(file_path)
-    df['Identifier'] = df.apply(extract_id, axis=1)
-    return df
+    lf = pl.scan_parquet(file_path)
+    schema = lf.collect_schema()
+    names = schema.names()
+
+    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in names]
+    if byte_cols:
+        lf = lf.with_columns([
+            pl.col(c).str.to_integer(base=16, strict=False).cast(pl.Int16).alias(c)
+            for c in byte_cols
+        ])
+
+    id_col = 'ID' if 'ID' in names else 'Identifier'
+    id_expr = pl.col(id_col).cast(pl.Utf8)
+
+    if 'j1939_metadata' in names:
+        try:
+            lf = lf.with_columns(
+                pl.when(pl.col('j1939_metadata').is_not_null())
+                .then(pl.lit('PGN: ') + pl.col('j1939_metadata').struct.field('PGN').cast(pl.Utf8))
+                .otherwise(pl.lit('ID: ') + id_expr)
+                .alias('Identifier')
+            )
+        except Exception:
+            lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
+    else:
+        lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
+
+    return lf.collect().to_pandas()