Normalize CAN IDs to strictly match same number of bit

stat/frequency.py

@@ -3,21 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import argparse
+import numbers
 from pathlib import Path
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 from utils.extractor import load_data
 
-def calc_freq(df: pd.DataFrame) -> pd.DataFrame:
+def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     """Calculates frequency counts and percentages for identifiers."""
-    freq_df = df['Identifier'].value_counts().reset_index()
+
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+
+    def format_can_id(x):
+        if pd.isna(x):
+            return "UNKNOWN"
+        if isinstance(x, numbers.Number):
+            return f"{int(x):08X}"
+        s = str(x).strip()
+        if s.lower().startswith('0x'):
+            s = s[2:]
+        if s.isdigit():
+            return f"{int(s):08X}"
+        try:
+            return f"{int(s, 16):08X}"
+        except ValueError:
+            return s
+
+    raw_ids = df[can_id_col]
+    formatted_ids = raw_ids.apply(format_can_id)
+
+    freq_df = formatted_ids.value_counts().reset_index()
+
     freq_df.columns = ['Identifier', 'Count']
+
     total = freq_df['Count'].sum()
     freq_df['Percentage'] = (freq_df['Count'] / total * 100).round(2)
+
     return freq_df.sort_values('Count', ascending=True)
 
-def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
+def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
     """Generates interactive horizontal bar chart with log x-axis."""
     fig = px.bar(
         stats_df, y='Identifier', x='Count', orientation='h', title=title, log_x=True,
@@ -43,7 +68,6 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
         ),
         yaxis=dict(
             title=dict(text="PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
-            #autorange="",
             showgrid=False,
             linecolor="#bdbdbd",
             tickfont=dict(size=12, color="#2a2a2a"),
@@ -51,6 +75,7 @@ def plot_freq(stats_df: pd.DataFrame, title: str) -> go.Figure:
             ticklen=4,
             tickcolor="#cccccc",
             automargin=True,
+            type='category' # Force categorical axis to ensure every ID gets a tick
         ),
         font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
         hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI",
@@ -104,8 +129,8 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     df = load_data(args.input)
-    stats = calc_freq(df)
+    stats = calculate_frequency(df)
-    fig = plot_freq(stats, title=args.title)
+    fig = plot_frequency(stats, title=args.title)
     config = {
         'responsive': True,
         'displaylogo': False,