diff --git a/stat/utils/extractor.py b/stat/utils/extractor.py new file mode 100644 index 0000000..a5dd172 --- /dev/null +++ b/stat/utils/extractor.py @@ -0,0 +1,23 @@ +import json +from pathlib import Path +import pandas as pd + +def extract_id(row: pd.Series) -> str: + """Extracts PGN from metadata or falls back to CAN ID.""" + meta = row.get('j1939_metadata') + if pd.isna(meta): + return f"ID: {row['ID']}" + if isinstance(meta, str): + try: + meta = json.loads(meta) + except json.JSONDecodeError: + return f"ID: {row['ID']}" + if isinstance(meta, dict) and 'PGN' in meta: + return f"PGN: {meta['PGN']}" + return f"ID: {row['ID']}" + +def load_data(file_path: Path) -> pd.DataFrame: + """Loads Parquet file and adds an Identifier column.""" + df = pd.read_parquet(file_path) + df['Identifier'] = df.apply(extract_id, axis=1) + return df