Use better function names

parser.py

@@ -19,7 +19,7 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
     out1_file = Path(out_bus1)
     out2_file = Path(out_bus2)
 
-    start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{1,8})\s+([0-9A-Fa-f]{1,2})\s+')
+    start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{7,8})\s+([0-9A-Fa-f]{1,2})\s+')
     byte_pattern = re.compile(r'^[0-9A-Fa-f]{2}$')
 
     with input_file.open('r', encoding='utf-8') as f_in, \
@@ -35,7 +35,12 @@ def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> N
         for line in f_in:
             for match in start_pattern.finditer(line):
                 bus = match.group(1)
-                can_id = match.group(2).upper()
+
+                can_id = match.group(2).upper().zfill(8)
+
+                if int(can_id, 16) > 0x1FFFFFFF:
+                    continue
+
                 dlc_str = match.group(3)
 
                 try:
@@ -116,4 +121,3 @@ if __name__ == '__main__':
         print(f"[*] Processing {args.input_csv}...")
         lf = parse_csv(args.input_csv)
         lf.sink_parquet(args.output_parquet)
-        print(f"[+] Saved parquet file to {args.output_parquet}")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "CANveyor"
-version = "0.0.1"
+version = "0.0.4"
 description = "J1939 CAN bus parser that works in pair with CANdigger"
 readme = "README.md"
 requires-python = ">=3.14"

stat/correlation.py

@@ -10,19 +10,21 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from utils.extractor import load_data
+from utils.extractor import to_int
 
+def _format_can_id(x):
+    """Safely cleans CAN ID strings without altering their length or value."""
+    if pd.isna(x):
+        return "UNKNOWN"
 
-def _to_int(x):
-    """Convert a hex string or integer to int, returning NaN on failure."""
-    if isinstance(x, (int, np.integer)):
-        return int(x)
-    if isinstance(x, str):
-        try:
-            return int(x, 16)
-        except ValueError:
-            return np.nan
-    return np.nan
+    s = str(x).strip()
+    if not s:
+        return "UNKNOWN"
 
+    if s.lower().startswith('0x'):
+        s = s[2:]
+
+    return s.upper()
 
 def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
     """Calculates inter-byte correlation grouped by identifier."""
@@ -32,12 +34,16 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
 
-    df_bytes = df[["Identifier"] + available_cols].copy()
-    for col in available_cols:
-        df_bytes[col] = df_bytes[col].apply(_to_int)
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    identifiers = df[can_id_col].apply(_format_can_id)
 
-    if target_id:
-        group = df_bytes[df_bytes["Identifier"] == target_id]
+    df_bytes = df[available_cols].copy()
+    for col in available_cols:
+        df_bytes[col] = df_bytes[col].apply(to_int)
+
+    if target_id is not None:
+        target_id = _format_can_id(target_id)
+        group = df_bytes[identifiers == target_id]
         if group.empty:
             raise ValueError(f"Identifier '{target_id}' not found in data")
         return group[available_cols].corr(method=method).fillna(0.0)
@@ -47,7 +53,9 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
         np.fill_diagonal(corr_arr, 0.0)
         return pd.Series(corr_arr.max(axis=0), index=group.columns).fillna(0.0)
 
-    return df_bytes.groupby("Identifier")[available_cols].apply(max_abs_corr)
+    result = df_bytes.groupby(identifiers)[available_cols].apply(max_abs_corr)
+    result.index.name = 'Identifier'
+    return result
 
 
 def plot_correlation_heatmap(corr_df: pd.DataFrame, target_id: str | None, title: str) -> go.Figure:

stat/entropy.py

@@ -10,19 +10,21 @@ import pandas as pd
 import plotly.graph_objects as go
 
 from utils.extractor import load_data
+from utils.extractor import to_int
 
+def _format_can_id(x):
+    """Safely cleans CAN ID strings without altering their length or value."""
+    if pd.isna(x):
+        return "UNKNOWN"
 
-def _to_int(x):
-    """Convert a hex string or integer to int, returning NaN on failure."""
-    if isinstance(x, (int, np.integer)):
-        return int(x)
-    if isinstance(x, str):
-        try:
-            return int(x, 16)
-        except ValueError:
-            return np.nan
-    return np.nan
+    s = str(x).strip()
+    if not s:
+        return "UNKNOWN"
 
+    if s.lower().startswith('0x'):
+        s = s[2:]
+
+    return s.upper()
 
 def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     """Calculates Shannon entropy per byte position for each identifier."""
@@ -31,9 +33,12 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
 
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    identifiers = df[can_id_col].apply(_format_can_id)
+
     df_bytes = df[available_cols].copy()
     for col in available_cols:
-        df_bytes[col] = df_bytes[col].apply(_to_int)
+        df_bytes[col] = df_bytes[col].apply(to_int)
 
     def entropy(s: pd.Series) -> float:
         s = s.dropna()
@@ -42,7 +47,9 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
         p = s.value_counts(normalize=True)
         return -np.sum(p * np.log2(p))
 
-    return df.groupby("Identifier")[available_cols].agg(entropy)
+    result = df_bytes.groupby(identifiers)[available_cols].agg(entropy)
+    result.index.name = 'Identifier'
+    return result
 
 
 def plot_entropy_heatmap(entropy_df: pd.DataFrame, title: str) -> go.Figure:

stat/frequency.py

@@ -9,15 +9,34 @@ import plotly.express as px
 import plotly.graph_objects as go
 from utils.extractor import load_data
 
-def calc_freq(df: pd.DataFrame) -> pd.DataFrame:
+def _format_can_id(x):
+    """Safely cleans CAN ID strings without altering their length or value."""
+    if pd.isna(x):
+        return "UNKNOWN"
+
+    s = str(x).strip()
+    if not s:
+        return "UNKNOWN"
+
+    if s.lower().startswith('0x'):
+        s = s[2:]
+
+    return s.upper()
+
+def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     """Calculates frequency counts and percentages for identifiers."""
-    freq_df = df['Identifier'].value_counts().reset_index()
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+
+    df['Formatted_ID'] = df[can_id_col].apply(_format_can_id)
+
+    freq_df = df['Formatted_ID'].value_counts().reset_index()
     freq_df.columns = ['Identifier', 'Count']
+
     total = freq_df['Count'].sum()
     freq_df['Percentage'] = (freq_df['Count'] / total * 100).round(2)
     return freq_df.sort_values('Count', ascending=True)
 
-def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
+def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
     """Generates interactive horizontal bar chart with log x-axis."""
     fig = px.bar(
         stats_df, y='Identifier', x='Count', orientation='h', title=title, log_x=True,
@@ -43,7 +62,6 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
         ),
         yaxis=dict(
             title=dict(text="PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
-            #autorange="",
             showgrid=False,
             linecolor="#bdbdbd",
             tickfont=dict(size=12, color="#2a2a2a"),
@@ -51,6 +69,7 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
             ticklen=4,
             tickcolor="#cccccc",
             automargin=True,
+            type='category'
         ),
         font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
         hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI",
@@ -104,8 +123,8 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     df = load_data(args.input)
-    stats = calc_freq(df)
-    fig = plot_freq(stats, title=args.title)
+    stats = calculate_frequency(df)
+    fig = plot_frequency(stats, title=args.title)
     config = {
         'responsive': True,
         'displaylogo': False,

stat/utils/extractor.py

@@ -1,3 +1,7 @@
+# File: extractor.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 import json
 from pathlib import Path