import json from pathlib import Path import numpy as np import pandas as pd def to_int(x): """Convert a hex string or integer to int, returning NaN on failure.""" if isinstance(x, (int, np.integer)): return int(x) if isinstance(x, str): try: return int(x, 16) except ValueError: return np.nan return np.nan def extract_id(row: pd.Series) -> str: """Extracts PGN from metadata or falls back to CAN ID.""" meta = row.get('j1939_metadata') if pd.isna(meta): return f"ID: {row['ID']}" if isinstance(meta, str): try: meta = json.loads(meta) except json.JSONDecodeError: return f"ID: {row['ID']}" if isinstance(meta, dict) and 'PGN' in meta: return f"PGN: {meta['PGN']}" return f"ID: {row['ID']}" def load_data(file_path: Path) -> pd.DataFrame: """Loads Parquet file and adds an Identifier column.""" df = pd.read_parquet(file_path) df['Identifier'] = df.apply(extract_id, axis=1) return df