CANveyor/main.py

# File: main.py
# Copyright (C) 2026 Erick Ahmed
# SPDX-License-Identifier: AGPL-3.0-or-later

import os
from pathlib import Path

import polars as pl
import dash
from dash import dcc, html, Input, Output
import dash_bootstrap_components as dbc
import numpy as np

from parser import parse_log, parse_csv
from decoder import decode_j1939_frames
from stats.utils.extractor import load_data
from stats.id_viewer import _format_can_id_vec, plot_bits
from stats.frequency import calculate_frequency, plot_frequency
from stats.correlation import calculate_correlation, plot_correlation_heatmap
from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap

RAW_LOG = "data/logs/rawlog.txt"
BUS1_CSV = "data/csv/bus1.csv"
BUS2_CSV = "data/csv/bus2.csv"
BUS1_PARQUET = "data/parquet/bus1.parquet"
BUS2_PARQUET = "data/parquet/bus2.parquet"
BUS1_DECODED = "data/parquet/bus1_decoded.parquet"
BUS2_DECODED = "data/parquet/bus2_decoded.parquet"

def run_pipeline():
    os.makedirs("data/logs", exist_ok=True)
    os.makedirs("data/csv", exist_ok=True)
    os.makedirs("data/parquet", exist_ok=True)
    if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
        print("Parsing raw log...")
        parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)

        print("Converting to parquet...")
        lf1 = parse_csv(BUS1_CSV)
        lf1.sink_parquet(BUS1_PARQUET)
        lf2 = parse_csv(BUS2_CSV)
        lf2.sink_parquet(BUS2_PARQUET)

        print("Decoding J1939...")
        df1 = pl.read_parquet(BUS1_PARQUET)
        df2 = pl.read_parquet(BUS2_PARQUET)
        dec1 = decode_j1939_frames(df1)
        dec2 = decode_j1939_frames(df2)
        dec1.write_parquet(BUS1_DECODED)
        dec2.write_parquet(BUS2_DECODED)

run_pipeline()

print("Loading data into memory...")
DATA = {
    "Bus 1": load_data(BUS1_DECODED),
    "Bus 2": load_data(BUS2_DECODED)
}

PRECOMPUTED_FIGURES = {}
DATA_BY_ID = {}
CORR_CACHE = {}

for bus, df in DATA.items():
    PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
    PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")

    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
    formatted = _format_can_id_vec(df[can_id_col])
    df = df.assign(Formatted_ID=formatted)

    df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')

    grouped = {}
    for can_id, group in df.groupby('Formatted_ID'):
        byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns]
        if not group.empty and len(byte_cols) > 0:
            arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False)
            if len(arr) > 1:
                changed = np.any(arr[1:] != arr[:-1], axis=1)
                keep = np.concatenate(([True], changed))
                group = group.iloc[keep]
        grouped[can_id] = (group, byte_cols)

    DATA_BY_ID[bus] = grouped

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
app.config.suppress_callback_exceptions = True

app.layout = dbc.Container([
    html.H1("CAN Bus Analyzer", className="my-4"),
    dbc.Row([
        dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
        dbc.Col(dcc.Dropdown(
            id='bus-selector',
            options=[{'label': k, 'value': k} for k in DATA.keys()],
            value='Bus 1',
            clearable=False
        ), width=2),
    ], className="mb-3"),
    dbc.Tabs([
        dbc.Tab(label="Frequency", tab_id="freq"),
        dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
        dbc.Tab(label="Correlation", tab_id="corr"),
        dbc.Tab(label="Entropy", tab_id="entropy"),
    ], id="tabs", active_tab="freq"),
    html.Div(id="tab-content", className="mt-3")
], fluid=True)

@app.callback(
    Output('tab-content', 'children'),
    Input('tabs', 'active_tab'),
    Input('bus-selector', 'value')
)
def render_content(tab, bus):
    df = DATA[bus]

    if tab == 'freq':
        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_freq"], style={'height': '80vh'})

    elif tab == 'id_viewer':
        ids = sorted(DATA_BY_ID[bus].keys())
        return html.Div([
            html.Label("Select CAN ID:"),
            dcc.Dropdown(
                id='id-selector',
                options=[{'label': i, 'value': i} for i in ids],
                value=ids[0] if ids else None,
                clearable=False,
                style={'width': '50%', 'marginBottom': '10px'}
            ),
            dcc.Graph(id='id-viewer-graph', style={'height': '70vh'})
        ])

    elif tab == 'corr':
        ids = sorted(DATA_BY_ID[bus].keys())
        return html.Div([
            dbc.Row([
                dbc.Col(html.Label("Method:"), width=1, className="mt-2"),
                dbc.Col(dcc.Dropdown(
                    id='corr-method',
                    options=[{'label': 'Pearson', 'value': 'pearson'}, {'label': 'Spearman', 'value': 'spearman'}],
                    value='pearson',
                    clearable=False
                ), width=2),
                dbc.Col(html.Label("Target ID:"), width=1, className="mt-2"),
                dbc.Col(dcc.Dropdown(
                    id='corr-target',
                    options=[{'label': 'All IDs (Max Corr)', 'value': 'all'}] + [{'label': i, 'value': i} for i in ids],
                    value='all',
                    clearable=True
                ), width=4),
            ], className="mb-3"),
            dcc.Graph(id='corr-graph', style={'height': '80vh'})
        ])

    elif tab == 'entropy':
        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_entropy"], style={'height': '80vh'})

    return html.Div("Tab not found")

@app.callback(
    Output('id-viewer-graph', 'figure'),
    Input('id-selector', 'value'),
    Input('bus-selector', 'value'),
    Input('tabs', 'active_tab'),
)
def update_id_viewer(selected_id, bus, tab):
    if tab != 'id_viewer' or not selected_id:
        return dash.no_update

    grouped_data = DATA_BY_ID.get(bus, {})
    if selected_id not in grouped_data:
        return dash.no_update

    filtered_df, byte_cols = grouped_data[selected_id]
    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization")

@app.callback(
    Output('corr-graph', 'figure'),
    Input('corr-method', 'value'),
    Input('corr-target', 'value'),
    Input('bus-selector', 'value'),
    Input('tabs', 'active_tab'),
)
def update_corr(method, target, bus, tab):
    if tab != 'corr':
        return dash.no_update

    target_id = None if target == 'all' or not target else target
    cache_key = (bus, method, target_id)

    if cache_key not in CORR_CACHE:
        df = DATA[bus]
        corr_df = calculate_correlation(df, method=method, target_id=target_id)
        CORR_CACHE[cache_key] = corr_df
    else:
        corr_df = CORR_CACHE[cache_key]

    title = f"{bus} Correlation"
    if target_id:
        title += f" ({target_id})"

    return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)

if __name__ == '__main__':
    app.run(debug=True)