Refactor statistical analysis modules for performance

- Optimize data processing pipelines across files by replacing iterative
  pandas operations with vectorized NumPy routines

stat/frequency.py

@@ -4,52 +4,57 @@
 
 import argparse
 from pathlib import Path
+import numpy as np
 import pandas as pd
-import plotly.express as px
 import plotly.graph_objects as go
 from utils.extractor import load_data
 
-def _format_can_id(x):
-    """Safely cleans CAN ID strings without altering their length or value."""
-    if pd.isna(x):
-        return "UNKNOWN"
-
-    s = str(x).strip()
-    if not s:
-        return "UNKNOWN"
-
-    if s.lower().startswith('0x'):
-        s = s[2:]
-
-    return s.upper()
+def _format_can_id_vec(s: pd.Series) -> pd.Series:
+    s = s.astype('string').str.strip()
+    s = s.str.replace(r'^0x', '', case=False, regex=True)
+    s = s.str.upper()
+    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
 
 def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
-    """Calculates frequency counts and percentages for identifiers."""
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    formatted = _format_can_id_vec(df[can_id_col])
+    df['Formatted_ID'] = formatted
 
-    df['Formatted_ID'] = df[can_id_col].apply(_format_can_id)
-
-    freq_df = df['Formatted_ID'].value_counts().reset_index()
-    freq_df.columns = ['Identifier', 'Count']
-
-    total = freq_df['Count'].sum()
-    freq_df['Percentage'] = (freq_df['Count'] / total * 100).round(2)
-    return freq_df.sort_values('Count', ascending=True)
+    counts = formatted.value_counts()
+    freq_df = pd.DataFrame({
+        'Identifier': counts.index,
+        'Count': counts.to_numpy(),
+    })
+    total = counts.sum()
+    freq_df['Percentage'] = np.round(freq_df['Count'] / total * 100, 2) if total else 0.0
+    return freq_df.sort_values('Count', ascending=True).reset_index(drop=True)
 
 def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
-    """Generates interactive horizontal bar chart with log x-axis."""
-    fig = px.bar(
-        stats_df, y='Identifier', x='Count', orientation='h', title=title, log_x=True,
-        labels={'Identifier': 'PGN / CAN ID', 'Count': 'Message Count'},
-        color='Count', color_continuous_scale='Turbo',
-        range_color=(stats_df['Count'].min(), stats_df['Count'].max()),
-        hover_data={'Percentage': ':.2f', 'Count': ':,', 'Identifier': True}
-    )
+    n = len(stats_df)
+    fig = go.Figure(go.Bar(
+        y=stats_df['Identifier'],
+        x=stats_df['Count'],
+        orientation='h',
+        marker=dict(
+            color=stats_df['Count'],
+            colorscale='Turbo',
+            cmin=int(stats_df['Count'].min()) if n else 0,
+            cmax=int(stats_df['Count'].max()) if n else 1,
+            line_width=0,
+        ),
+        customdata=stats_df[['Percentage']].to_numpy(),
+        hovertemplate="<b>%{y}</b><br>Count: %{x:,}<br>Share: %{customdata[0]}%<extra></extra>",
+        texttemplate='%{x:,}',
+        textposition='outside',
+        cliponaxis=False,
+    ))
+
     fig.update_layout(
-        height=max(600, len(stats_df) * 18),
+        height=max(600, n * 18),
         autosize=True,
         template='plotly_white',
         xaxis=dict(
+            type='log',
             title=dict(text="Message count [log scale]", font=dict(size=13, color="#1a1a1a")),
             side="top",
             dtick=1,
@@ -69,11 +74,10 @@ def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
             ticklen=4,
             tickcolor="#cccccc",
             automargin=True,
-            type='category'
+            type='category',
         ),
         font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
-        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI",
-                        bordercolor='#cccccc'),
+        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI", bordercolor='#cccccc'),
         margin=dict(l=200, r=40, t=120, b=60),
         bargap=0.35,
         coloraxis_colorbar=dict(
@@ -87,39 +91,27 @@ def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
             yanchor='bottom',
             tickformat=',',
             outlinecolor='#cccccc',
-            outlinewidth=0.5
+            outlinewidth=0.5,
         ),
-        title=dict(font=dict(size=20, color='#1a1a1a'), x=0.5, xanchor='center',
-                   pad=dict(b=20))
+        title=dict(text=title, font=dict(size=20, color='#1a1a1a'), x=0.5, xanchor='center', pad=dict(b=20)),
     )
     fig.update_xaxes(
         showgrid=True, gridwidth=0.5, gridcolor='#e8e8e8',
         zeroline=False, linecolor='#bdbdbd', mirror=False,
         tickformat=',',
-        minor=dict(showgrid=True, gridcolor='#f4f4f4', gridwidth=0.5)
+        minor=dict(showgrid=True, gridcolor='#f4f4f4', gridwidth=0.5),
     )
     fig.update_yaxes(
         showgrid=False, zeroline=False, linecolor='#bdbdbd',
-        ticks='outside', ticklen=4, tickcolor='#cccccc',
-        automargin=True
-    )
-    fig.update_traces(
-        hovertemplate="<b>%{y}</b><br>Count: %{x:,}<br>Share: %{customdata[0]}%<extra></extra>",
-        marker_line_width=0,
-        texttemplate='%{x:,}',
-        textposition='outside',
-        textfont=dict(size=10, color='#666666'),
-        cliponaxis=False,
-        selected=dict(marker=dict(opacity=0.6)),
-        unselected=dict(marker=dict(opacity=0.2))
+        ticks='outside', ticklen=4, tickcolor='#cccccc', automargin=True,
     )
     return fig
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Analyze CAN bus message frequency")
     parser.add_argument("input", type=Path, help="Path to the input CAN log file")
-    parser.add_argument("output", type=Path, nargs="?", default=Path("freq_report.html"), help="Path to the output HTML report")
-    parser.add_argument("title", nargs="?", default="Frequency", help="Title for the HTML report")
+    parser.add_argument("output", type=Path, nargs="?", default=Path("freq_report.html"))
+    parser.add_argument("title", nargs="?", default="Frequency")
     args = parser.parse_args()
 
     df = load_data(args.input)
@@ -130,6 +122,6 @@ if __name__ == "__main__":
         'displaylogo': False,
         'scrollZoom': True,
         'modeBarButtonsToAdd': ['toggleSpikelines'],
-        'toImageButtonOptions': {'format': 'png', 'scale': 2}
+        'toImageButtonOptions': {'format': 'png', 'scale': 2},
     }
     fig.write_html(str(args.output), include_plotlyjs='cdn', config=config)