diff --git a/main.py b/main.py index 2249de6..9d16086 100644 --- a/main.py +++ b/main.py @@ -2,50 +2,167 @@ # Copyright (C) 2026 Erick Ahmed # SPDX-License-Identifier: AGPL-3.0-or-later -import diskcache -import flask_caching +import os +from pathlib import Path + +import polars as pl import dash -from dash import Input, Output, State, dcc, html, no_update +from dash import dcc, html, Input, Output import dash_bootstrap_components as dbc -CACHE_DATA_DIR = ".cache_data" -background_callback_manager = dash.DiskcacheManager(cache_dir=CACHE_DATA_DIR) -data_cache = flask_caching.Cache(config={'CACHE_TYPE': 'FileSystemCache', 'CACHE_DIR': CACHE_DATA_DIR}) +from parser import parse_log, parse_csv +from decoder import decode_j1939_frames +from stats.utils.extractor import load_data -app = dash.Dash( - __name__, - external_stylesheets=[dbc.themes.BOOTSTRAP], - background_callback_manager=background_callback_manager -) -data_cache.init_app(app.server) +from stats.frequency import calculate_frequency, plot_frequency +from stats.id_viewer import prepare_data, plot_bits +from stats.correlation import calculate_correlation, plot_correlation_heatmap +from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap -def _get_df(session_data): - if not session_data or "token" not in session_data: - return None - return data_cache.get(session_data["token"]) +RAW_LOG = "data/logs/rawlog.txt" +BUS1_CSV = "data/csv/bus1.csv" +BUS2_CSV = "data/csv/bus2.csv" +BUS1_PARQUET = "data/parquet/bus1.parquet" +BUS2_PARQUET = "data/parquet/bus2.parquet" +BUS1_DECODED = "data/parquet/bus1_decoded.parquet" +BUS2_DECODED = "data/parquet/bus2_decoded.parquet" + +def run_pipeline(): + os.makedirs("data/logs", exist_ok=True) + if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists(): + print("Parsing raw log...") + parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV) + + print("Converting to parquet...") + lf1 = parse_csv(BUS1_CSV) + lf1.sink_parquet(BUS1_PARQUET) + lf2 = parse_csv(BUS2_CSV) + lf2.sink_parquet(BUS2_PARQUET) + + print("Decoding J1939...") + df1 = pl.read_parquet(BUS1_PARQUET) + df2 = pl.read_parquet(BUS2_PARQUET) + dec1 = decode_j1939_frames(df1) + dec2 = decode_j1939_frames(df2) + dec1.write_parquet(BUS1_DECODED) + dec2.write_parquet(BUS2_DECODED) + +run_pipeline() + +print("Loading data into memory...") +DATA = { + "Bus 1": load_data(BUS1_DECODED), + "Bus 2": load_data(BUS2_DECODED) +} + +app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) + +app.layout = dbc.Container([ + html.H1("CAN Bus Analyzer", className="my-4"), + dbc.Row([ + dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"), + dbc.Col(dcc.Dropdown( + id='bus-selector', + options=[{'label': k, 'value': k} for k in DATA.keys()], + value='Bus 1', + clearable=False + ), width=2), + ], className="mb-3"), + dbc.Tabs([ + dbc.Tab(label="Frequency", tab_id="freq"), + dbc.Tab(label="ID Viewer", tab_id="id_viewer"), + dbc.Tab(label="Correlation", tab_id="corr"), + dbc.Tab(label="Entropy", tab_id="entropy"), + ], id="tabs", active_tab="freq"), + html.Div(id="tab-content", className="mt-3") +], fluid=True) @app.callback( - Output("graph-correlation", "figure"), - Input("corr-method", "value"), - Input("corr-target", "value"), - Input("session-store", "data"), - - background=True, - prevent_initial_call=True, + Output('tab-content', 'children'), + Input('tabs', 'active_tab'), + Input('bus-selector', 'value') ) -def update_correlation(method, target, session_data): - df = _get_df(session_data) - if df is None or not method: - return no_update +def render_content(tab, bus): + df = DATA[bus] - target_id = None if target == "all" else target - try: - corr_df = calculate_correlation(df, method=method, target_id=target_id) - title = f"Inter-Byte Correlation ({method.capitalize()})" - if target_id: - title += f" - {target_id}" - return plot_correlation_heatmap(corr_df, target_id=target_id, title=title) - except Exception as exc: - fig = dash.go.Figure() - fig.update_layout(title=f"Error: {exc}") - return fig + if tab == 'freq': + stats = calculate_frequency(df) + fig = plot_frequency(stats, title=f"{bus} Frequency") + return dcc.Graph(figure=fig, style={'height': '80vh'}) + + elif tab == 'id_viewer': + ids = sorted(df['ID'].unique().tolist()) + return html.Div([ + html.Label("Select CAN ID:"), + dcc.Dropdown( + id='id-selector', + options=[{'label': i, 'value': i} for i in ids], + value=ids[0] if ids else None, + clearable=False, + style={'width': '50%', 'marginBottom': '10px'} + ), + dcc.Graph(id='id-viewer-graph', style={'height': '70vh'}) + ]) + + elif tab == 'corr': + ids = sorted(df['ID'].unique().tolist()) + return html.Div([ + dbc.Row([ + dbc.Col(html.Label("Method:"), width=1, className="mt-2"), + dbc.Col(dcc.Dropdown( + id='corr-method', + options=[{'label': 'Pearson', 'value': 'pearson'}, {'label': 'Spearman', 'value': 'spearman'}], + value='pearson', + clearable=False + ), width=2), + dbc.Col(html.Label("Target ID:"), width=1, className="mt-2"), + dbc.Col(dcc.Dropdown( + id='corr-target', + options=[{'label': 'All IDs (Max Corr)', 'value': 'all'}] + [{'label': i, 'value': i} for i in ids], + value='all', + clearable=True + ), width=4), + ], className="mb-3"), + dcc.Graph(id='corr-graph', style={'height': '80vh'}) + ]) + + elif tab == 'entropy': + entropy_df = calculate_byte_entropy(df) + fig = plot_entropy_heatmap(entropy_df, title=f"{bus} Byte-Level Entropy") + return dcc.Graph(figure=fig, style={'height': '80vh'}) + + return html.Div("Tab not found") + +@app.callback( + Output('id-viewer-graph', 'figure'), + Input('id-selector', 'value'), + Input('bus-selector', 'value'), + Input('tabs', 'active_tab'), +) +def update_id_viewer(selected_id, bus, tab): + if tab != 'id_viewer' or not selected_id: + return dash.no_update + df = DATA[bus] + filtered_df, byte_cols = prepare_data(df, selected_id) + return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization") + +@app.callback( + Output('corr-graph', 'figure'), + Input('corr-method', 'value'), + Input('corr-target', 'value'), + Input('bus-selector', 'value'), + Input('tabs', 'active_tab'), +) +def update_corr(method, target, bus, tab): + if tab != 'corr': + return dash.no_update + df = DATA[bus] + target_id = None if target == 'all' or not target else target + corr_df = calculate_correlation(df, method=method, target_id=target_id) + title = f"{bus} Correlation" + if target_id: + title += f" ({target_id})" + return plot_correlation_heatmap(corr_df, target_id=target_id, title=title) + +if __name__ == '__main__': + app.run(debug=True)