# File: main.py # Copyright (C) 2026 Erick Ahmed # SPDX-License-Identifier: AGPL-3.0-or-later import os from pathlib import Path import polars as pl import dash from dash import dcc, html, Input, Output import dash_bootstrap_components as dbc import numpy as np from parser import parse_log, parse_csv from decoder import decode_j1939_frames from stats.utils.extractor import load_data from stats.id_viewer import _format_can_id_vec, plot_bits from stats.frequency import calculate_frequency, plot_frequency from stats.correlation import calculate_correlation, plot_correlation_heatmap from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap RAW_LOG = "data/logs/rawlog.txt" BUS1_CSV = "data/csv/bus1.csv" BUS2_CSV = "data/csv/bus2.csv" BUS1_PARQUET = "data/parquet/bus1.parquet" BUS2_PARQUET = "data/parquet/bus2.parquet" BUS1_DECODED = "data/parquet/bus1_decoded.parquet" BUS2_DECODED = "data/parquet/bus2_decoded.parquet" def run_pipeline(): os.makedirs("data/logs", exist_ok=True) os.makedirs("data/csv", exist_ok=True) os.makedirs("data/parquet", exist_ok=True) if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists(): print("Parsing raw log...") parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV) print("Converting to parquet...") parse_csv(BUS1_CSV).sink_parquet(BUS1_PARQUET) parse_csv(BUS2_CSV).sink_parquet(BUS2_PARQUET) print("Decoding J1939...") df1 = pl.read_parquet(BUS1_PARQUET) df2 = pl.read_parquet(BUS2_PARQUET) dec1 = decode_j1939_frames(df1) dec2 = decode_j1939_frames(df2) dec1.write_parquet(BUS1_DECODED) dec2.write_parquet(BUS2_DECODED) run_pipeline() print("Loading data into memory...") DATA = { "Bus 1": load_data(BUS1_DECODED), "Bus 2": load_data(BUS2_DECODED) } PRECOMPUTED_FIGURES = {} DATA_BY_ID = {} CORR_CACHE = {} for bus, df in DATA.items(): PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency") PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy") can_id_col = 'ID' if 'ID' in df.columns else 'Identifier' formatted = _format_can_id_vec(df[can_id_col]) df = df.assign(Formatted_ID=formatted) df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable') grouped = {} for can_id, group in df.groupby(by='Formatted_ID'): byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns] if not group.empty and len(byte_cols) > 0: arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False) if len(arr) > 1: changed = np.any(arr[1:] != arr[:-1], axis=1) keep = np.concatenate(([True], changed)) group = group.iloc[keep] grouped[can_id] = (group, byte_cols) DATA_BY_ID[bus] = grouped app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) app.config.suppress_callback_exceptions = True app.layout = dbc.Container([ html.H1("CAN Bus Analyzer", className="my-4"), dbc.Row([ dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"), dbc.Col(dcc.Dropdown( id='bus-selector', options=[{'label': k, 'value': k} for k in DATA.keys()], value='Bus 1', clearable=False ), width=2), ], className="mb-3"), dbc.Tabs([ dbc.Tab(label="Frequency", tab_id="freq"), dbc.Tab(label="ID Viewer", tab_id="id_viewer"), dbc.Tab(label="Correlation", tab_id="corr"), dbc.Tab(label="Entropy", tab_id="entropy"), ], id="tabs", active_tab="freq"), html.Div(id="tab-content", className="mt-3") ], fluid=True) @app.callback( Output('tab-content', 'children'), Input('tabs', 'active_tab'), Input('bus-selector', 'value') ) def render_content(tab, bus): df = DATA[bus] if tab == 'freq': return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_freq"], style={'height': '80vh'}) elif tab == 'id_viewer': ids = sorted(DATA_BY_ID[bus].keys()) return html.Div([ html.Label("Select CAN ID:"), dcc.Dropdown( id='id-selector', options=[{'label': i, 'value': i} for i in ids], value=ids[0] if ids else None, clearable=False, style={'width': '50%', 'marginBottom': '10px'} ), dcc.Graph(id='id-viewer-graph', style={'height': '70vh'}) ]) elif tab == 'corr': ids = sorted(DATA_BY_ID[bus].keys()) return html.Div([ dbc.Row([ dbc.Col(html.Label("Method:"), width=1, className="mt-2"), dbc.Col(dcc.Dropdown( id='corr-method', options=[{'label': 'Pearson', 'value': 'pearson'}, {'label': 'Spearman', 'value': 'spearman'}], value='pearson', clearable=False ), width=2), dbc.Col(html.Label("Target ID:"), width=1, className="mt-2"), dbc.Col(dcc.Dropdown( id='corr-target', options=[{'label': 'All IDs (Max Corr)', 'value': 'all'}] + [{'label': i, 'value': i} for i in ids], value='all', clearable=True ), width=4), ], className="mb-3"), dcc.Graph(id='corr-graph', style={'height': '80vh'}) ]) elif tab == 'entropy': return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_entropy"], style={'height': '80vh'}) return html.Div("Tab not found") @app.callback( Output('id-viewer-graph', 'figure'), Input('id-selector', 'value'), Input('bus-selector', 'value'), Input('tabs', 'active_tab'), ) def update_id_viewer(selected_id, bus, tab): if tab != 'id_viewer' or not selected_id: return dash.no_update grouped_data = DATA_BY_ID.get(bus, {}) if selected_id not in grouped_data: return dash.no_update filtered_df, byte_cols = grouped_data[selected_id] return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization") @app.callback( Output('corr-graph', 'figure'), Input('corr-method', 'value'), Input('corr-target', 'value'), Input('bus-selector', 'value'), Input('tabs', 'active_tab'), ) def update_corr(method, target, bus, tab): if tab != 'corr': return dash.no_update target_id = None if target == 'all' or not target else target cache_key = (bus, method, target_id) if cache_key not in CORR_CACHE: df = DATA[bus] corr_df = calculate_correlation(df, method=method, target_id=target_id) CORR_CACHE[cache_key] = corr_df else: corr_df = CORR_CACHE[cache_key] title = f"{bus} Correlation" if target_id: title += f" ({target_id})" return plot_correlation_heatmap(corr_df, target_id=target_id, title=title) if __name__ == '__main__': app.run(debug=True)