Merge pull request 'Implement Plotly Dash app with efficient dynamic resampler' (#6) from dev-dash into main

Reviewed-on: erickahmed/CANveyor#6

main.py

@@ -4,6 +4,7 @@
 
 import os
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import polars as pl
 import dash
@@ -29,15 +30,15 @@ BUS2_DECODED = "data/parquet/bus2_decoded.parquet"
 
 def run_pipeline():
     os.makedirs("data/logs", exist_ok=True)
+    os.makedirs("data/csv", exist_ok=True)
+    os.makedirs("data/parquet", exist_ok=True)
     if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
         print("Parsing raw log...")
         parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)
 
         print("Converting to parquet...")
-        lf1 = parse_csv(BUS1_CSV)
-        lf1.sink_parquet(BUS1_PARQUET)
-        lf2 = parse_csv(BUS2_CSV)
-        lf2.sink_parquet(BUS2_PARQUET)
+        parse_csv(BUS1_CSV).sink_parquet(BUS1_PARQUET)
+        parse_csv(BUS2_CSV).sink_parquet(BUS2_PARQUET)
 
         print("Decoding J1939...")
         df1 = pl.read_parquet(BUS1_PARQUET)
@@ -59,9 +60,10 @@ PRECOMPUTED_FIGURES = {}
 DATA_BY_ID = {}
 CORR_CACHE = {}
 
-for bus, df in DATA.items():
-    PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
-    PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")
+def process_bus_data(bus, df):
+    precomp = {}
+    precomp[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
+    precomp[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")
 
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
@@ -70,7 +72,7 @@ for bus, df in DATA.items():
     df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')
 
     grouped = {}
-    for can_id, group in df.groupby('Formatted_ID'):
+    for can_id, group in df.groupby(by='Formatted_ID'):
         byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns]
         if not group.empty and len(byte_cols) > 0:
             arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False)
@@ -80,29 +82,44 @@ for bus, df in DATA.items():
                 group = group.iloc[keep]
         grouped[can_id] = (group, byte_cols)
 
-    DATA_BY_ID[bus] = grouped
+    return precomp, grouped
+
+with ThreadPoolExecutor() as executor:
+    futures = {executor.submit(process_bus_data, bus, df): bus for bus, df in DATA.items()}
+    for future in futures:
+        bus = futures[future]
+        precomp, grouped = future.result()
+        PRECOMPUTED_FIGURES.update(precomp)
+        DATA_BY_ID[bus] = grouped
 
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
 app.config.suppress_callback_exceptions = True
 
 app.layout = dbc.Container([
-    html.H1("CAN Bus Analyzer", className="my-4"),
-    dbc.Row([
-        dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
-        dbc.Col(dcc.Dropdown(
-            id='bus-selector',
-            options=[{'label': k, 'value': k} for k in DATA.keys()],
-            value='Bus 1',
-            clearable=False
-        ), width=2),
-    ], className="mb-3"),
+    html.H1("CANveyor", className="my-4"),
     dbc.Tabs([
-        dbc.Tab(label="Frequency", tab_id="freq"),
-        dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
-        dbc.Tab(label="Correlation", tab_id="corr"),
-        dbc.Tab(label="Entropy", tab_id="entropy"),
-    ], id="tabs", active_tab="freq"),
-    html.Div(id="tab-content", className="mt-3")
+        dbc.Tab(label="Overview", tab_id="overview", children=[
+            html.Div(id="overview-content")
+        ]),
+        dbc.Tab(label="Statistics", tab_id="statistics", children=[
+            dbc.Row([
+                dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
+                dbc.Col(dcc.Dropdown(
+                    id='bus-selector',
+                    options=[{'label': k, 'value': k} for k in DATA.keys()],
+                    value='Bus 1',
+                    clearable=False
+                ), width=2),
+            ], className="mb-3 mt-3"),
+            dbc.Tabs([
+                dbc.Tab(label="Frequency", tab_id="freq"),
+                dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
+                dbc.Tab(label="Correlation", tab_id="corr"),
+                dbc.Tab(label="Entropy", tab_id="entropy"),
+            ], id="tabs", active_tab="freq"),
+            html.Div(id="tab-content", className="mt-3")
+        ])
+    ], id="main-tabs", active_tab="statistics")
 ], fluid=True)
 
 @app.callback(

pyproject.toml

@@ -1,7 +1,15 @@
 [project]
 name = "CANveyor"
-version = "0.0.4"
+version = "0.1.0"
 description = "J1939 CAN bus parser that works in pair with CANdigger"
 readme = "README.md"
-requires-python = ">=3.14"
-dependencies = ["polars", "pathlib", "typing"]
+requires-python = ">=3.10"
+dependencies = [
+    "polars",
+    "dash",
+    "dash-bootstrap-components",
+    "numpy",
+    "pandas",
+    "plotly",
+    "plotly-resampler"
+]

stats/correlation.py

@@ -4,6 +4,7 @@
 
 import argparse
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -27,8 +28,7 @@ def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
     return df
 
 def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
-    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
-    available_cols = [col for col in byte_cols if col in df.columns]
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
 
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
@@ -70,8 +70,7 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
     else:
         groups = []
 
-    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
-    for gi, sub in enumerate(groups):
+    def _process_group(sub):
         mask = ~np.isnan(sub).any(axis=1)
         sub = sub[mask]
         if len(sub) > 1:
@@ -81,7 +80,11 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
                 c = np.abs(np.corrcoef(sub, rowvar=False))
             np.nan_to_num(c, copy=False, nan=0.0)
             np.fill_diagonal(c, 0.0)
-            out[gi] = c.max(axis=0)
+            return c.max(axis=0)
+        return np.zeros(n_cols, dtype=np.float64)
+
+    with ThreadPoolExecutor() as executor:
+        out = np.array(list(executor.map(_process_group, groups)))
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/entropy.py

@@ -4,6 +4,7 @@
 
 import argparse
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -36,8 +37,7 @@ def _entropy_col(a: np.ndarray) -> float:
     return float(-np.sum(p * np.log2(p)))
 
 def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
-    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
-    available_cols = byte_cols
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
 
@@ -64,10 +64,14 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     else:
         groups = []
 
-    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
-    for gi, sub in enumerate(groups):
+    def _process_group(sub):
+        res = np.zeros(n_cols, dtype=np.float64)
         for ci in range(n_cols):
-            out[gi, ci] = _entropy_col(sub[:, ci])
+            res[ci] = _entropy_col(sub[:, ci])
+        return res
+
+    with ThreadPoolExecutor() as executor:
+        out = np.array(list(executor.map(_process_group, groups)))
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/frequency.py

@@ -18,7 +18,6 @@ def _format_can_id_vec(s: pd.Series) -> pd.Series:
 def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
-    df['Formatted_ID'] = formatted
 
     counts = formatted.value_counts()
     freq_df = pd.DataFrame({