From d274897cf353a7533ac3b462e720d329487d879a Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Wed, 22 Jul 2026 18:07:33 +0200 Subject: [PATCH 1/5] Implement lttbc --- stats/id_viewer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/stats/id_viewer.py b/stats/id_viewer.py index 59f00ac..4c9ed00 100644 --- a/stats/id_viewer.py +++ b/stats/id_viewer.py @@ -7,6 +7,7 @@ from pathlib import Path import numpy as np import pandas as pd import plotly.graph_objects as go +import lttbc from stats.utils.extractor import load_data def _format_can_id_vec(s: pd.Series) -> pd.Series: @@ -43,13 +44,20 @@ def plot_bits(df, byte_cols, can_id, title): fig = go.Figure() colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf'] n = len(byte_cols) + max_points = 2000 x = df['Timestamp'].to_numpy() if not df.empty else np.array([]) for i, col in enumerate(byte_cols): y = df[col].to_numpy(dtype=np.float32, copy=False) if not df.empty else np.array([]) + + if len(x) > max_points and len(x) == len(y): + x_plot, y_plot = lttbc.downsample(x, y, max_points) + else: + x_plot, y_plot = x, y + fig.add_trace(go.Scattergl( - x=x, - y=y, + x=x_plot, + y=y_plot, mode='lines', line=dict(shape='hv', width=2, color=colors[i % len(colors)]), name=col.upper(), -- 2.52.0 From 02e46ddf0b8286356d9af72e8f28bc63233a91c5 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Wed, 22 Jul 2026 18:11:35 +0200 Subject: [PATCH 2/5] Implement plotly-resampler --- stats/id_viewer.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/stats/id_viewer.py b/stats/id_viewer.py index 4c9ed00..104f162 100644 --- a/stats/id_viewer.py +++ b/stats/id_viewer.py @@ -7,7 +7,7 @@ from pathlib import Path import numpy as np import pandas as pd import plotly.graph_objects as go -import lttbc +from plotly_resampler import FigureResampler from stats.utils.extractor import load_data def _format_can_id_vec(s: pd.Series) -> pd.Series: @@ -41,29 +41,21 @@ def prepare_data(df, target_id): return filtered, byte_cols def plot_bits(df, byte_cols, can_id, title): - fig = go.Figure() + fig = FigureResampler() colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf'] n = len(byte_cols) - max_points = 2000 x = df['Timestamp'].to_numpy() if not df.empty else np.array([]) for i, col in enumerate(byte_cols): y = df[col].to_numpy(dtype=np.float32, copy=False) if not df.empty else np.array([]) - if len(x) > max_points and len(x) == len(y): - x_plot, y_plot = lttbc.downsample(x, y, max_points) - else: - x_plot, y_plot = x, y - - fig.add_trace(go.Scattergl( - x=x_plot, - y=y_plot, + fig.add_trace(go.Scatter( mode='lines', line=dict(shape='hv', width=2, color=colors[i % len(colors)]), name=col.upper(), legendgroup=col.upper(), hovertemplate=f"{col.upper()}
Time: %{{x}}
Value: %{{y}}", - )) + ), hf_x=x, hf_y=y) all_button = dict(label='ALL', method='restyle', args=[{'visible': [True] * n}]) none_button = dict(label='NONE', method='restyle', args=[{'visible': ['legendonly'] * n}]) -- 2.52.0 From 93e0e3f64828fa0bfc4200c686cb1ce4e0018722 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Wed, 22 Jul 2026 18:16:43 +0200 Subject: [PATCH 3/5] Suppress callback exceptions --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 9d16086..d5c8d84 100644 --- a/main.py +++ b/main.py @@ -56,6 +56,7 @@ DATA = { } app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) +app.config.suppress_callback_exceptions = True app.layout = dbc.Container([ html.H1("CAN Bus Analyzer", className="my-4"), -- 2.52.0 From 2da646fa8008c11106f55032b62d89734ac11de3 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Wed, 22 Jul 2026 18:20:18 +0200 Subject: [PATCH 4/5] Precompute CAN data - Slower startup - Much faster visualization (from O(n) to O(1)) --- main.py | 64 ++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/main.py b/main.py index d5c8d84..ed45176 100644 --- a/main.py +++ b/main.py @@ -9,13 +9,13 @@ import polars as pl import dash from dash import dcc, html, Input, Output import dash_bootstrap_components as dbc +import numpy as np from parser import parse_log, parse_csv from decoder import decode_j1939_frames from stats.utils.extractor import load_data - +from stats.id_viewer import _format_can_id_vec, plot_bits from stats.frequency import calculate_frequency, plot_frequency -from stats.id_viewer import prepare_data, plot_bits from stats.correlation import calculate_correlation, plot_correlation_heatmap from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap @@ -55,6 +55,33 @@ DATA = { "Bus 2": load_data(BUS2_DECODED) } +PRECOMPUTED_FIGURES = {} +DATA_BY_ID = {} +CORR_CACHE = {} + +for bus, df in DATA.items(): + PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency") + PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy") + + can_id_col = 'ID' if 'ID' in df.columns else 'Identifier' + formatted = _format_can_id_vec(df[can_id_col]) + df = df.assign(Formatted_ID=formatted) + + df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable') + + grouped = {} + for can_id, group in df.groupby('Formatted_ID'): + byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns] + if not group.empty and len(byte_cols) > 0: + arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False) + if len(arr) > 1: + changed = np.any(arr[1:] != arr[:-1], axis=1) + keep = np.concatenate(([True], changed)) + group = group.iloc[keep] + grouped[can_id] = (group, byte_cols) + + DATA_BY_ID[bus] = grouped + app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) app.config.suppress_callback_exceptions = True @@ -87,12 +114,10 @@ def render_content(tab, bus): df = DATA[bus] if tab == 'freq': - stats = calculate_frequency(df) - fig = plot_frequency(stats, title=f"{bus} Frequency") - return dcc.Graph(figure=fig, style={'height': '80vh'}) + return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_freq"], style={'height': '80vh'}) elif tab == 'id_viewer': - ids = sorted(df['ID'].unique().tolist()) + ids = sorted(DATA_BY_ID[bus].keys()) return html.Div([ html.Label("Select CAN ID:"), dcc.Dropdown( @@ -106,7 +131,7 @@ def render_content(tab, bus): ]) elif tab == 'corr': - ids = sorted(df['ID'].unique().tolist()) + ids = sorted(DATA_BY_ID[bus].keys()) return html.Div([ dbc.Row([ dbc.Col(html.Label("Method:"), width=1, className="mt-2"), @@ -128,9 +153,7 @@ def render_content(tab, bus): ]) elif tab == 'entropy': - entropy_df = calculate_byte_entropy(df) - fig = plot_entropy_heatmap(entropy_df, title=f"{bus} Byte-Level Entropy") - return dcc.Graph(figure=fig, style={'height': '80vh'}) + return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_entropy"], style={'height': '80vh'}) return html.Div("Tab not found") @@ -143,8 +166,12 @@ def render_content(tab, bus): def update_id_viewer(selected_id, bus, tab): if tab != 'id_viewer' or not selected_id: return dash.no_update - df = DATA[bus] - filtered_df, byte_cols = prepare_data(df, selected_id) + + grouped_data = DATA_BY_ID.get(bus, {}) + if selected_id not in grouped_data: + return dash.no_update + + filtered_df, byte_cols = grouped_data[selected_id] return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization") @app.callback( @@ -157,12 +184,21 @@ def update_id_viewer(selected_id, bus, tab): def update_corr(method, target, bus, tab): if tab != 'corr': return dash.no_update - df = DATA[bus] + target_id = None if target == 'all' or not target else target - corr_df = calculate_correlation(df, method=method, target_id=target_id) + cache_key = (bus, method, target_id) + + if cache_key not in CORR_CACHE: + df = DATA[bus] + corr_df = calculate_correlation(df, method=method, target_id=target_id) + CORR_CACHE[cache_key] = corr_df + else: + corr_df = CORR_CACHE[cache_key] + title = f"{bus} Correlation" if target_id: title += f" ({target_id})" + return plot_correlation_heatmap(corr_df, target_id=target_id, title=title) if __name__ == '__main__': -- 2.52.0 From c06d813c26caf4ca289084c743fdc1677e2086f4 Mon Sep 17 00:00:00 2001 From: Erick Ahmed Date: Wed, 22 Jul 2026 18:21:32 +0200 Subject: [PATCH 5/5] Remove resampling information on legend --- stats/id_viewer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stats/id_viewer.py b/stats/id_viewer.py index 104f162..f3e30c8 100644 --- a/stats/id_viewer.py +++ b/stats/id_viewer.py @@ -41,7 +41,10 @@ def prepare_data(df, target_id): return filtered, byte_cols def plot_bits(df, byte_cols, can_id, title): - fig = FigureResampler() + fig = FigureResampler( + resampled_trace_prefix_suffix=("", ""), + show_mean_aggregation_size=False + ) colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf'] n = len(byte_cols) -- 2.52.0