Implement Pearson and Spearman per-bit correleration #3

Merged
eeeck merged 4 commits from dev-inter-byte-correlation into main 2026-07-13 21:54:40 +02:00
Showing only changes of commit 5e6f81b50b - Show all commits
+13 -15
View File
@@ -1,11 +1,20 @@
# File: extractor.py
# Copyright (C) 2026 Erick Ahmed
# SPDX-License-Identifier: AGPL-3.0-or-later
import json
from pathlib import Path
import numpy as np
import pandas as pd
def to_int(x):
"""Convert a hex string or integer to int, returning NaN on failure."""
if isinstance(x, (int, np.integer)):
return int(x)
if isinstance(x, str):
try:
return int(x, 16)
except ValueError:
return np.nan
return np.nan
def extract_id(row: pd.Series) -> str:
"""Extracts PGN from metadata or falls back to CAN ID."""
meta = row.get('j1939_metadata')
@@ -25,14 +34,3 @@ def load_data(file_path: Path) -> pd.DataFrame:
df = pd.read_parquet(file_path)
df['Identifier'] = df.apply(extract_id, axis=1)
return df
def to_int(x):
"""Convert a hex string or integer to int, returning NaN on failure."""
if isinstance(x, (int, np.integer)):
return int(x)
if isinstance(x, str):
try:
return int(x, 16)
except ValueError:
return np.nan
return np.nan