eeeck/CANveyor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 6 Wiki Activity

Normalize CAN IDs number of bits

Browse Source
This commit is contained in:
Erick
2026-07-14 00:03:34 +02:00
parent 9ddc6ea0d5
commit d8105e2da3
4 changed files with 72 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parser.py
+7 -3
View File
@@ -19,7 +19,7 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
out1_file = Path(out_bus1)
out2_file = Path(out_bus2)
start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{1,8})\s+([0-9A-Fa-f]{1,2})\s+')
start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{7,8})\s+([0-9A-Fa-f]{1,2})\s+')
byte_pattern = re.compile(r'^[0-9A-Fa-f]{2}$')
with input_file.open('r', encoding='utf-8') as f_in, \
@@ -35,7 +35,12 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
for line in f_in:
for match in start_pattern.finditer(line):
bus = match.group(1)
can_id = match.group(2).upper()
can_id = match.group(2).upper().zfill(8)
if int(can_id, 16) > 0x1FFFFFFF:
continue
dlc_str = match.group(3)
try:
@@ -116,4 +121,3 @@ if __name__ == '__main__':
print(f"[*] Processing {args.input_csv}...")
lf = parse_csv(args.input_csv)
lf.sink_parquet(args.output_parquet)
print(f"[+] Saved parquet file to {args.output_parquet}")
stat/correlation.py
+24 -4
View File
@@ -12,6 +12,20 @@ import plotly.graph_objects as go
from utils.extractor import load_data
from utils.extractor import to_int
def _format_can_id(x):
"""Safely cleans CAN ID strings without altering their length or value."""
if pd.isna(x):
return "UNKNOWN"
s = str(x).strip()
if not s:
return "UNKNOWN"
if s.lower().startswith('0x'):
s = s[2:]
return s.upper()
def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
"""Calculates inter-byte correlation grouped by identifier."""
byte_cols = [f"b{i}" for i in range(8)]
@@ -20,12 +34,16 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
if not available_cols:
raise ValueError("No byte columns (b0-b7) found in the DataFrame")
df_bytes = df[["Identifier"] + available_cols].copy()
can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
identifiers = df[can_id_col].apply(_format_can_id)
df_bytes = df[available_cols].copy()
for col in available_cols:
df_bytes[col] = df_bytes[col].apply(to_int)
if target_id:
group = df_bytes[df_bytes["Identifier"] == target_id]
if target_id is not None:
target_id = _format_can_id(target_id)
group = df_bytes[identifiers == target_id]
if group.empty:
raise ValueError(f"Identifier '{target_id}' not found in data")
return group[available_cols].corr(method=method).fillna(0.0)
@@ -35,7 +53,9 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
np.fill_diagonal(corr_arr, 0.0)
return pd.Series(corr_arr.max(axis=0), index=group.columns).fillna(0.0)
return df_bytes.groupby("Identifier")[available_cols].apply(max_abs_corr)
result = df_bytes.groupby(identifiers)[available_cols].apply(max_abs_corr)
result.index.name = 'Identifier'
return result
def plot_correlation_heatmap(corr_df: pd.DataFrame, target_id: str | None, title: str) -> go.Figure:
stat/entropy.py
+20 -1
View File
@@ -12,6 +12,20 @@ import plotly.graph_objects as go
from utils.extractor import load_data
from utils.extractor import to_int
def _format_can_id(x):
"""Safely cleans CAN ID strings without altering their length or value."""
if pd.isna(x):
return "UNKNOWN"
s = str(x).strip()
if not s:
return "UNKNOWN"
if s.lower().startswith('0x'):
s = s[2:]
return s.upper()
def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
"""Calculates Shannon entropy per byte position for each identifier."""
byte_cols = [f"b{i}" for i in range(8)]
@@ -19,6 +33,9 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
if not available_cols:
raise ValueError("No byte columns (b0-b7) found in the DataFrame")
can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
identifiers = df[can_id_col].apply(_format_can_id)
df_bytes = df[available_cols].copy()
for col in available_cols:
df_bytes[col] = df_bytes[col].apply(to_int)
@@ -30,7 +47,9 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
p = s.value_counts(normalize=True)
return -np.sum(p * np.log2(p))
return df.groupby("Identifier")[available_cols].agg(entropy)
result = df_bytes.groupby(identifiers)[available_cols].agg(entropy)
result.index.name = 'Identifier'
return result
def plot_entropy_heatmap(entropy_df: pd.DataFrame, title: str) -> go.Figure:
stat/frequency.py
+21 -2
View File
@@ -9,10 +9,29 @@ import plotly.express as px
import plotly.graph_objects as go
from utils.extractor import load_data
def _format_can_id(x):
"""Safely cleans CAN ID strings without altering their length or value."""
if pd.isna(x):
return "UNKNOWN"
s = str(x).strip()
if not s:
return "UNKNOWN"
if s.lower().startswith('0x'):
s = s[2:]
return s.upper()
def calc_freq(df: pd.DataFrame) -> pd.DataFrame:
"""Calculates frequency counts and percentages for identifiers."""
freq_df = df['Identifier'].value_counts().reset_index()
can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
df['Formatted_ID'] = df[can_id_col].apply(_format_can_id)
freq_df = df['Formatted_ID'].value_counts().reset_index()
freq_df.columns = ['Identifier', 'Count']
total = freq_df['Count'].sum()
freq_df['Percentage'] = (freq_df['Count'] / total * 100).round(2)
return freq_df.sort_values('Count', ascending=True)
@@ -43,7 +62,6 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
),
yaxis=dict(
title=dict(text="PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
#autorange="",
showgrid=False,
linecolor="#bdbdbd",
tickfont=dict(size=12, color="#2a2a2a"),
@@ -51,6 +69,7 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
ticklen=4,
tickcolor="#cccccc",
automargin=True,
type='category'
),
font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI",