Refactor main application to use Polars pipeline
- replaced the caching layer with a pre-processing pipeline that parses raw logs into decoded Parquet files
This commit is contained in:
@@ -2,50 +2,167 @@
|
||||
# Copyright (C) 2026 Erick Ahmed
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import diskcache
|
||||
import flask_caching
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
import dash
|
||||
from dash import Input, Output, State, dcc, html, no_update
|
||||
from dash import dcc, html, Input, Output
|
||||
import dash_bootstrap_components as dbc
|
||||
|
||||
CACHE_DATA_DIR = ".cache_data"
|
||||
background_callback_manager = dash.DiskcacheManager(cache_dir=CACHE_DATA_DIR)
|
||||
data_cache = flask_caching.Cache(config={'CACHE_TYPE': 'FileSystemCache', 'CACHE_DIR': CACHE_DATA_DIR})
|
||||
from parser import parse_log, parse_csv
|
||||
from decoder import decode_j1939_frames
|
||||
from stats.utils.extractor import load_data
|
||||
|
||||
app = dash.Dash(
|
||||
__name__,
|
||||
external_stylesheets=[dbc.themes.BOOTSTRAP],
|
||||
background_callback_manager=background_callback_manager
|
||||
)
|
||||
data_cache.init_app(app.server)
|
||||
from stats.frequency import calculate_frequency, plot_frequency
|
||||
from stats.id_viewer import prepare_data, plot_bits
|
||||
from stats.correlation import calculate_correlation, plot_correlation_heatmap
|
||||
from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap
|
||||
|
||||
def _get_df(session_data):
|
||||
if not session_data or "token" not in session_data:
|
||||
return None
|
||||
return data_cache.get(session_data["token"])
|
||||
RAW_LOG = "data/logs/rawlog.txt"
|
||||
BUS1_CSV = "data/csv/bus1.csv"
|
||||
BUS2_CSV = "data/csv/bus2.csv"
|
||||
BUS1_PARQUET = "data/parquet/bus1.parquet"
|
||||
BUS2_PARQUET = "data/parquet/bus2.parquet"
|
||||
BUS1_DECODED = "data/parquet/bus1_decoded.parquet"
|
||||
BUS2_DECODED = "data/parquet/bus2_decoded.parquet"
|
||||
|
||||
def run_pipeline():
|
||||
os.makedirs("data/logs", exist_ok=True)
|
||||
if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
|
||||
print("Parsing raw log...")
|
||||
parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)
|
||||
|
||||
print("Converting to parquet...")
|
||||
lf1 = parse_csv(BUS1_CSV)
|
||||
lf1.sink_parquet(BUS1_PARQUET)
|
||||
lf2 = parse_csv(BUS2_CSV)
|
||||
lf2.sink_parquet(BUS2_PARQUET)
|
||||
|
||||
print("Decoding J1939...")
|
||||
df1 = pl.read_parquet(BUS1_PARQUET)
|
||||
df2 = pl.read_parquet(BUS2_PARQUET)
|
||||
dec1 = decode_j1939_frames(df1)
|
||||
dec2 = decode_j1939_frames(df2)
|
||||
dec1.write_parquet(BUS1_DECODED)
|
||||
dec2.write_parquet(BUS2_DECODED)
|
||||
|
||||
run_pipeline()
|
||||
|
||||
print("Loading data into memory...")
|
||||
DATA = {
|
||||
"Bus 1": load_data(BUS1_DECODED),
|
||||
"Bus 2": load_data(BUS2_DECODED)
|
||||
}
|
||||
|
||||
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
||||
|
||||
app.layout = dbc.Container([
|
||||
html.H1("CAN Bus Analyzer", className="my-4"),
|
||||
dbc.Row([
|
||||
dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
|
||||
dbc.Col(dcc.Dropdown(
|
||||
id='bus-selector',
|
||||
options=[{'label': k, 'value': k} for k in DATA.keys()],
|
||||
value='Bus 1',
|
||||
clearable=False
|
||||
), width=2),
|
||||
], className="mb-3"),
|
||||
dbc.Tabs([
|
||||
dbc.Tab(label="Frequency", tab_id="freq"),
|
||||
dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
|
||||
dbc.Tab(label="Correlation", tab_id="corr"),
|
||||
dbc.Tab(label="Entropy", tab_id="entropy"),
|
||||
], id="tabs", active_tab="freq"),
|
||||
html.Div(id="tab-content", className="mt-3")
|
||||
], fluid=True)
|
||||
|
||||
@app.callback(
|
||||
Output("graph-correlation", "figure"),
|
||||
Input("corr-method", "value"),
|
||||
Input("corr-target", "value"),
|
||||
Input("session-store", "data"),
|
||||
|
||||
background=True,
|
||||
prevent_initial_call=True,
|
||||
Output('tab-content', 'children'),
|
||||
Input('tabs', 'active_tab'),
|
||||
Input('bus-selector', 'value')
|
||||
)
|
||||
def update_correlation(method, target, session_data):
|
||||
df = _get_df(session_data)
|
||||
if df is None or not method:
|
||||
return no_update
|
||||
def render_content(tab, bus):
|
||||
df = DATA[bus]
|
||||
|
||||
target_id = None if target == "all" else target
|
||||
try:
|
||||
corr_df = calculate_correlation(df, method=method, target_id=target_id)
|
||||
title = f"Inter-Byte Correlation ({method.capitalize()})"
|
||||
if target_id:
|
||||
title += f" - {target_id}"
|
||||
return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
|
||||
except Exception as exc:
|
||||
fig = dash.go.Figure()
|
||||
fig.update_layout(title=f"Error: {exc}")
|
||||
return fig
|
||||
if tab == 'freq':
|
||||
stats = calculate_frequency(df)
|
||||
fig = plot_frequency(stats, title=f"{bus} Frequency")
|
||||
return dcc.Graph(figure=fig, style={'height': '80vh'})
|
||||
|
||||
elif tab == 'id_viewer':
|
||||
ids = sorted(df['ID'].unique().tolist())
|
||||
return html.Div([
|
||||
html.Label("Select CAN ID:"),
|
||||
dcc.Dropdown(
|
||||
id='id-selector',
|
||||
options=[{'label': i, 'value': i} for i in ids],
|
||||
value=ids[0] if ids else None,
|
||||
clearable=False,
|
||||
style={'width': '50%', 'marginBottom': '10px'}
|
||||
),
|
||||
dcc.Graph(id='id-viewer-graph', style={'height': '70vh'})
|
||||
])
|
||||
|
||||
elif tab == 'corr':
|
||||
ids = sorted(df['ID'].unique().tolist())
|
||||
return html.Div([
|
||||
dbc.Row([
|
||||
dbc.Col(html.Label("Method:"), width=1, className="mt-2"),
|
||||
dbc.Col(dcc.Dropdown(
|
||||
id='corr-method',
|
||||
options=[{'label': 'Pearson', 'value': 'pearson'}, {'label': 'Spearman', 'value': 'spearman'}],
|
||||
value='pearson',
|
||||
clearable=False
|
||||
), width=2),
|
||||
dbc.Col(html.Label("Target ID:"), width=1, className="mt-2"),
|
||||
dbc.Col(dcc.Dropdown(
|
||||
id='corr-target',
|
||||
options=[{'label': 'All IDs (Max Corr)', 'value': 'all'}] + [{'label': i, 'value': i} for i in ids],
|
||||
value='all',
|
||||
clearable=True
|
||||
), width=4),
|
||||
], className="mb-3"),
|
||||
dcc.Graph(id='corr-graph', style={'height': '80vh'})
|
||||
])
|
||||
|
||||
elif tab == 'entropy':
|
||||
entropy_df = calculate_byte_entropy(df)
|
||||
fig = plot_entropy_heatmap(entropy_df, title=f"{bus} Byte-Level Entropy")
|
||||
return dcc.Graph(figure=fig, style={'height': '80vh'})
|
||||
|
||||
return html.Div("Tab not found")
|
||||
|
||||
@app.callback(
|
||||
Output('id-viewer-graph', 'figure'),
|
||||
Input('id-selector', 'value'),
|
||||
Input('bus-selector', 'value'),
|
||||
Input('tabs', 'active_tab'),
|
||||
)
|
||||
def update_id_viewer(selected_id, bus, tab):
|
||||
if tab != 'id_viewer' or not selected_id:
|
||||
return dash.no_update
|
||||
df = DATA[bus]
|
||||
filtered_df, byte_cols = prepare_data(df, selected_id)
|
||||
return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization")
|
||||
|
||||
@app.callback(
|
||||
Output('corr-graph', 'figure'),
|
||||
Input('corr-method', 'value'),
|
||||
Input('corr-target', 'value'),
|
||||
Input('bus-selector', 'value'),
|
||||
Input('tabs', 'active_tab'),
|
||||
)
|
||||
def update_corr(method, target, bus, tab):
|
||||
if tab != 'corr':
|
||||
return dash.no_update
|
||||
df = DATA[bus]
|
||||
target_id = None if target == 'all' or not target else target
|
||||
corr_df = calculate_correlation(df, method=method, target_id=target_id)
|
||||
title = f"{bus} Correlation"
|
||||
if target_id:
|
||||
title += f" ({target_id})"
|
||||
return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True)
|
||||
|
||||
Reference in New Issue
Block a user