Suppress callback references

logs/view.py

@@ -1,83 +0,0 @@
-# File: logs/view.py
-# Copyright (C) 2026 Erick Ahmed
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
-import pandas as pd
-from dash import html, dash_table, dcc
-import dash_bootstrap_components as dbc
-
-PAGE_SIZE = 25000
-
-def prepare_logs_data(df: pd.DataFrame) -> pd.DataFrame:
-    if df is None or df.empty:
-        return pd.DataFrame()
-
-    df = df.copy()
-
-    if 'j1939_metadata' in df.columns:
-        df['Priority'] = df['j1939_metadata'].apply(lambda x: x.get('Priority') if isinstance(x, dict) else None)
-        df['PF'] = df['j1939_metadata'].apply(lambda x: x.get('PF') if isinstance(x, dict) else None)
-        df['PS'] = df['j1939_metadata'].apply(lambda x: x.get('PS') if isinstance(x, dict) else None)
-        df['SA'] = df['j1939_metadata'].apply(lambda x: x.get('SA') if isinstance(x, dict) else None)
-        df['DA'] = df['j1939_metadata'].apply(lambda x: x.get('DA') if isinstance(x, dict) else None)
-        df['PGN'] = df['j1939_metadata'].apply(lambda x: x.get('PGN') if isinstance(x, dict) else None)
-    else:
-        for col in ['Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']:
-            df[col] = None
-
-    for i in range(8):
-        col = f'b{i}'
-        if col in df.columns:
-            df[col] = df[col].apply(lambda x: f"{int(x):02X}" if pd.notna(x) else "")
-        else:
-            df[col] = ""
-
-    if 'ID' in df.columns:
-        df['ID'] = df['ID'].astype(str)
-
-    display_cols = ['Timestamp', 'ID', 'DLC', 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6', 'b7', 'Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']
-    display_df = df[[c for c in display_cols if c in df.columns]]
-
-    return display_df.fillna("")
-
-def get_logs_table_component():
-    return html.Div([
-        html.Div(id='logs-info-text', className="text-muted mb-2"),
-        dash_table.DataTable(
-            id='logs-table',
-            virtualization=True,
-            page_action='none',
-            style_table={'overflowX': 'auto', 'height': '70vh', 'overflowY': 'auto'},
-            style_header={
-                'backgroundColor': '#1a1a1a',
-                'color': 'white',
-                'fontWeight': 'bold',
-                'textAlign': 'center',
-                'position': 'sticky',
-                'top': 0
-            },
-            style_data={
-                'backgroundColor': '#f8f9fa',
-                'color': '#2a2a2a',
-                'textAlign': 'center'
-            },
-            style_data_conditional=[
-                {
-                    'if': {'row_index': 'odd'},
-                    'backgroundColor': 'rgb(240, 240, 240)'
-                }
-            ],
-            style_cell={
-                'minWidth': '80px',
-                'padding': '5px',
-                'textAlign': 'center',
-                'fontFamily': 'Segoe UI, Arial, sans-serif'
-            }
-        ),
-        html.Div([
-            dbc.Button("Prev", id='logs-prev-btn', color="secondary", outline=True, size="sm", className="me-2"),
-            html.Div(id='logs-page-nav', className="d-inline-block", style={'verticalAlign': 'middle'}),
-            dbc.Button("Next", id='logs-next-btn', color="secondary", outline=True, size="sm", className="ms-2"),
-        ], className="d-flex justify-content-center align-items-center mt-3"),
-        dcc.Store(id='logs-current-page', data=0),
-    ])

main.py

@@ -4,308 +4,95 @@
 
 import os
 from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
 
 import polars as pl
 import dash
-from dash import dcc, html, Input, Output, State
+from dash import dcc, html, Input, Output
 import dash_bootstrap_components as dbc
-import numpy as np
 
 from parser import parse_log, parse_csv
 from decoder import decode_j1939_frames
 from stats.utils.extractor import load_data
-from stats.id_viewer import _format_can_id_vec, plot_bits
+
 from stats.frequency import calculate_frequency, plot_frequency
+from stats.id_viewer import prepare_data, plot_bits
 from stats.correlation import calculate_correlation, plot_correlation_heatmap
 from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap
-from logs.view import get_logs_table_component, prepare_logs_data
 
-RAW_LOG_DIR = "data/logs"
-PAGE_SIZE = 25000
-
-def parse_vehicle_from_filename(filename: str):
-    stem = Path(filename).stem
-    if '-' in stem:
-        brand, model_part = stem.split('-', 1)
-    else:
-        brand, model_part = stem, "Unknown"
-    model = model_part.replace('_', ' ')
-    vehicle = f"{brand} {model}".strip()
-    return vehicle, brand, model
+RAW_LOG = "data/logs/rawlog.txt"
+BUS1_CSV = "data/csv/bus1.csv"
+BUS2_CSV = "data/csv/bus2.csv"
+BUS1_PARQUET = "data/parquet/bus1.parquet"
+BUS2_PARQUET = "data/parquet/bus2.parquet"
+BUS1_DECODED = "data/parquet/bus1_decoded.parquet"
+BUS2_DECODED = "data/parquet/bus2_decoded.parquet"
 
 def run_pipeline():
-    os.makedirs(RAW_LOG_DIR, exist_ok=True)
-    os.makedirs("data/csv", exist_ok=True)
-    os.makedirs("data/parquet", exist_ok=True)
+    os.makedirs("data/logs", exist_ok=True)
+    if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
+        print("Parsing raw log...")
+        parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)
 
-    for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
-        vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
+        print("Converting to parquet...")
+        lf1 = parse_csv(BUS1_CSV)
+        lf1.sink_parquet(BUS1_PARQUET)
+        lf2 = parse_csv(BUS2_CSV)
+        lf2.sink_parquet(BUS2_PARQUET)
 
-        bus1_csv = f"data/csv/{vehicle}_bus1.csv"
-        bus2_csv = f"data/csv/{vehicle}_bus2.csv"
-        bus1_parquet = f"data/parquet/{vehicle}_bus1.parquet"
-        bus2_parquet = f"data/parquet/{vehicle}_bus2.parquet"
-        bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
-        bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
-
-        if not Path(bus1_decoded).exists() or not Path(bus2_decoded).exists():
-            print(f"Parsing raw log: {log_file.name}...")
-            parse_log(str(log_file), bus1_csv, bus2_csv)
-
-            print("Converting to parquet...")
-            parse_csv(bus1_csv).sink_parquet(bus1_parquet)
-            parse_csv(bus2_csv).sink_parquet(bus2_parquet)
-
-            print("Decoding J1939...")
-            df1 = pl.read_parquet(bus1_parquet)
-            df2 = pl.read_parquet(bus2_parquet)
-            dec1 = decode_j1939_frames(df1)
-            dec2 = decode_j1939_frames(df2)
-            dec1.write_parquet(bus1_decoded)
-            dec2.write_parquet(bus2_decoded)
+        print("Decoding J1939...")
+        df1 = pl.read_parquet(BUS1_PARQUET)
+        df2 = pl.read_parquet(BUS2_PARQUET)
+        dec1 = decode_j1939_frames(df1)
+        dec2 = decode_j1939_frames(df2)
+        dec1.write_parquet(BUS1_DECODED)
+        dec2.write_parquet(BUS2_DECODED)
 
 run_pipeline()
 
 print("Loading data into memory...")
-DATA = {}
-VEHICLE_META = {}
-
-for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
-    vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
-    VEHICLE_META[vehicle] = {"brand": brand, "model": model}
-
-    bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
-    bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
-
-    if Path(bus1_decoded).exists() and Path(bus2_decoded).exists():
-        DATA[vehicle] = {
-            "Bus 1": load_data(bus1_decoded),
-            "Bus 2": load_data(bus2_decoded)
-        }
-
-PRECOMPUTED_FIGURES = {}
-DATA_BY_ID = {}
-CORR_CACHE = {}
-PREPARED_LOGS_CACHE = {}
-
-def process_bus_data(vehicle, bus, df):
-    precomp = {}
-    precomp[f"{vehicle}_{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{vehicle} {bus} Frequency")
-    precomp[f"{vehicle}_{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{vehicle} {bus} Byte-Level Entropy")
-
-    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
-    formatted = _format_can_id_vec(df[can_id_col])
-    df = df.assign(Formatted_ID=formatted)
-    df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')
-
-    grouped = {}
-    for can_id, group in df.groupby(by='Formatted_ID'):
-        byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns]
-        if not group.empty and len(byte_cols) > 0:
-            arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False)
-            if len(arr) > 1:
-                changed = np.any(arr[1:] != arr[:-1], axis=1)
-                keep = np.concatenate(([True], changed))
-                group = group.iloc[keep]
-        grouped[can_id] = (group, byte_cols)
-
-    return vehicle, bus, precomp, grouped
-
-with ThreadPoolExecutor() as executor:
-    futures = []
-    for vehicle, buses in DATA.items():
-        for bus, df in buses.items():
-            futures.append(executor.submit(process_bus_data, vehicle, bus, df))
-    for future in futures:
-        v, b, precomp, grouped = future.result()
-        PRECOMPUTED_FIGURES.update(precomp)
-        DATA_BY_ID[(v, b)] = grouped
+DATA = {
+    "Bus 1": load_data(BUS1_DECODED),
+    "Bus 2": load_data(BUS2_DECODED)
+}
 
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
 app.config.suppress_callback_exceptions = True
 
 app.layout = dbc.Container([
-    html.H1("CANveyor", className="my-4"),
+    html.H1("CAN Bus Analyzer", className="my-4"),
+    dbc.Row([
+        dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
+        dbc.Col(dcc.Dropdown(
+            id='bus-selector',
+            options=[{'label': k, 'value': k} for k in DATA.keys()],
+            value='Bus 1',
+            clearable=False
+        ), width=2),
+    ], className="mb-3"),
     dbc.Tabs([
-        dbc.Tab(label="Overview", tab_id="overview", children=[
-            html.Div(id="overview-content")
-        ]),
-        dbc.Tab(label="Logs", tab_id="logs", children=[
-            dbc.Row([
-                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
-                dbc.Col(dcc.Dropdown(
-                    id='logs-vehicle-selector',
-                    options=[{'label': v, 'value': v} for v in DATA.keys()],
-                    value=list(DATA.keys())[0] if DATA else None,
-                    clearable=False
-                ), width=3, className="me-4"),
-                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
-                dbc.Col(dcc.Dropdown(
-                    id='logs-bus-selector',
-                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
-                    value='Bus 1',
-                    clearable=False
-                ), width=2),
-            ], className="mb-3 mt-3", align="end"),
-            get_logs_table_component()
-        ]),
-        dbc.Tab(label="Statistics", tab_id="statistics", children=[
-            dbc.Row([
-                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
-                dbc.Col(dcc.Dropdown(
-                    id='vehicle-selector',
-                    options=[{'label': v, 'value': v} for v in DATA.keys()],
-                    value=list(DATA.keys())[0] if DATA else None,
-                    clearable=False
-                ), width=3, className="me-4"),
-                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
-                dbc.Col(dcc.Dropdown(
-                    id='bus-selector',
-                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
-                    value='Bus 1',
-                    clearable=False
-                ), width=2),
-            ], className="mb-3 mt-3", align="end"),
-            dbc.Tabs([
-                dbc.Tab(label="Frequency", tab_id="freq"),
-                dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
-                dbc.Tab(label="Correlation", tab_id="corr"),
-                dbc.Tab(label="Entropy", tab_id="entropy"),
-            ], id="tabs", active_tab="freq"),
-            html.Div(id="tab-content", className="mt-3")
-        ])
-    ], id="main-tabs", active_tab="statistics")
+        dbc.Tab(label="Frequency", tab_id="freq"),
+        dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
+        dbc.Tab(label="Correlation", tab_id="corr"),
+        dbc.Tab(label="Entropy", tab_id="entropy"),
+    ], id="tabs", active_tab="freq"),
+    html.Div(id="tab-content", className="mt-3")
 ], fluid=True)
 
-def get_prepared_logs(vehicle, bus):
-    cache_key = (vehicle, bus)
-    if cache_key not in PREPARED_LOGS_CACHE:
-        df = DATA[vehicle][bus]
-        PREPARED_LOGS_CACHE[cache_key] = prepare_logs_data(df)
-    return PREPARED_LOGS_CACHE[cache_key]
-
-def build_page_buttons(current_page: int, total_pages: int, max_buttons: int = 15) -> list:
-    buttons: list = []
-    if total_pages <= 1:
-        return buttons
-
-    half = max_buttons // 2
-    start = max(0, current_page - half)
-    end = min(total_pages, start + max_buttons)
-    if end - start < max_buttons:
-        start = max(0, end - max_buttons)
-
-    if start > 0:
-        buttons.append(
-            dbc.Button("1", id={'type': 'page-btn', 'index': 0}, color="secondary", outline=True, size="sm", className="me-1")
-        )
-        if start > 1:
-            buttons.append(html.Span("…", className="mx-1 align-middle"))
-
-    for i in range(start, end):
-        is_current = (i == current_page)
-        buttons.append(
-            dbc.Button(
-                str(i + 1),
-                id={'type': 'page-btn', 'index': i},
-                size="sm",
-                color="primary" if is_current else "secondary",
-                outline=not is_current,
-                className="me-1",
-                disabled=is_current,
-            )
-        )
-
-    if end < total_pages:
-        if end < total_pages - 1:
-            buttons.append(html.Span("…", className="mx-1 align-middle"))
-        buttons.append(
-            dbc.Button(
-                str(total_pages),
-                id={'type': 'page-btn', 'index': total_pages - 1},
-                color="secondary", outline=True, size="sm", className="me-1",
-            )
-        )
-
-    return buttons
-
-@app.callback(
-    Output('logs-table', 'data'),
-    Output('logs-table', 'columns'),
-    Output('logs-info-text', 'children'),
-    Output('logs-page-nav', 'children'),
-    Output('logs-current-page', 'data'),
-    Input('logs-vehicle-selector', 'value'),
-    Input('logs-bus-selector', 'value'),
-    Input('logs-prev-btn', 'n_clicks'),
-    Input('logs-next-btn', 'n_clicks'),
-    Input({'type': 'page-btn', 'index': dash.ALL}, 'n_clicks'),
-    State('logs-current-page', 'data'),
-)
-def update_logs_table(vehicle, bus, prev_clicks, next_clicks, page_btn_clicks, current_page):
-    if (not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]):
-        return [], [], "No data available", [], 0
-
-    prepared_df = get_prepared_logs(vehicle, bus)
-    total_rows = len(prepared_df)
-
-    if total_rows == 0:
-        return [], [], "No data available", [], 0
-
-    total_pages = max(1, (total_rows + PAGE_SIZE - 1) // PAGE_SIZE)
-
-    ctx = dash.callback_context
-    triggered_id = ctx.triggered_id
-
-    current_page = current_page if current_page is not None else 0
-
-    if triggered_id in ('logs-vehicle-selector', 'logs-bus-selector'):
-        current_page = 0
-
-    elif triggered_id == 'logs-prev-btn':
-        current_page = max(0, current_page - 1)
-
-    elif triggered_id == 'logs-next-btn':
-        current_page = current_page + 1
-
-    elif isinstance(triggered_id, dict) and triggered_id.get('type') == 'page-btn':
-        if ctx.triggered and ctx.triggered[0]['value']:
-            current_page = triggered_id['index']
-
-    current_page = max(0, min(current_page, total_pages - 1))
-
-    start_idx = current_page * PAGE_SIZE
-    end_idx = min(start_idx + PAGE_SIZE, total_rows)
-    page_data = prepared_df.iloc[start_idx:end_idx].to_dict('records')
-
-    columns = [{"name": i, "id": i} for i in prepared_df.columns]
-
-    info_text = (f"Page {current_page + 1} of {total_pages} | "
-                 f"Showing rows {start_idx + 1:,}–{end_idx:,} "
-                 f"of {total_rows:,} total frames")
-
-    page_buttons = build_page_buttons(current_page, total_pages)
-
-    return page_data, columns, info_text, page_buttons, current_page
-
 @app.callback(
     Output('tab-content', 'children'),
     Input('tabs', 'active_tab'),
-    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value')
 )
-def render_content(tab, vehicle, bus):
-    if not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]:
-        return html.Div("No data available")
-
-    df = DATA[vehicle][bus]
+def render_content(tab, bus):
+    df = DATA[bus]
 
     if tab == 'freq':
-        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_freq"], style={'height': '80vh'})
+        stats = calculate_frequency(df)
+        fig = plot_frequency(stats, title=f"{bus} Frequency")
+        return dcc.Graph(figure=fig, style={'height': '80vh'})
 
     elif tab == 'id_viewer':
-        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
+        ids = sorted(df['ID'].unique().tolist())
         return html.Div([
             html.Label("Select CAN ID:"),
             dcc.Dropdown(
@@ -319,7 +106,7 @@ def render_content(tab, vehicle, bus):
         ])
 
     elif tab == 'corr':
-        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
+        ids = sorted(df['ID'].unique().tolist())
         return html.Div([
             dbc.Row([
                 dbc.Col(html.Label("Method:"), width=1, className="mt-2"),
@@ -341,55 +128,42 @@ def render_content(tab, vehicle, bus):
         ])
 
     elif tab == 'entropy':
-        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_entropy"], style={'height': '80vh'})
+        entropy_df = calculate_byte_entropy(df)
+        fig = plot_entropy_heatmap(entropy_df, title=f"{bus} Byte-Level Entropy")
+        return dcc.Graph(figure=fig, style={'height': '80vh'})
 
     return html.Div("Tab not found")
 
 @app.callback(
     Output('id-viewer-graph', 'figure'),
     Input('id-selector', 'value'),
-    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value'),
     Input('tabs', 'active_tab'),
 )
-def update_id_viewer(selected_id, vehicle, bus, tab):
-    if tab != 'id_viewer' or not selected_id or not vehicle or not bus:
+def update_id_viewer(selected_id, bus, tab):
+    if tab != 'id_viewer' or not selected_id:
         return dash.no_update
-
-    grouped_data = DATA_BY_ID.get((vehicle, bus), {})
-    if selected_id not in grouped_data:
-        return dash.no_update
-
-    filtered_df, byte_cols = grouped_data[selected_id]
-    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{vehicle} {bus} Byte Visualization")
+    df = DATA[bus]
+    filtered_df, byte_cols = prepare_data(df, selected_id)
+    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization")
 
 @app.callback(
     Output('corr-graph', 'figure'),
     Input('corr-method', 'value'),
     Input('corr-target', 'value'),
-    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value'),
     Input('tabs', 'active_tab'),
 )
-def update_corr(method, target, vehicle, bus, tab):
-    if tab != 'corr' or not vehicle or not bus:
+def update_corr(method, target, bus, tab):
+    if tab != 'corr':
         return dash.no_update
-
+    df = DATA[bus]
     target_id = None if target == 'all' or not target else target
-    cache_key = (vehicle, bus, method, target_id)
-
-    if cache_key not in CORR_CACHE:
-        df = DATA[vehicle][bus]
-        corr_df = calculate_correlation(df, method=method, target_id=target_id)
-        CORR_CACHE[cache_key] = corr_df
-    else:
-        corr_df = CORR_CACHE[cache_key]
-
-    title = f"{vehicle} {bus} Correlation"
+    corr_df = calculate_correlation(df, method=method, target_id=target_id)
+    title = f"{bus} Correlation"
     if target_id:
         title += f" ({target_id})"
-
     return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
 
 if __name__ == '__main__':
-    app.run(debug=False)
+    app.run(debug=True)

pyproject.toml

@@ -1,15 +1,7 @@
 [project]
 name = "CANveyor"
-version = "0.1.0"
+version = "0.0.4"
 description = "J1939 CAN bus parser that works in pair with CANdigger"
 readme = "README.md"
-requires-python = ">=3.10"
-dependencies = [
-    "polars",
-    "dash",
-    "dash-bootstrap-components",
-    "numpy",
-    "pandas",
-    "plotly",
-    "plotly-resampler"
-]
+requires-python = ">=3.14"
+dependencies = ["polars", "pathlib", "typing"]

stats/correlation.py

@@ -4,7 +4,6 @@
 
 import argparse
 from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -28,7 +27,8 @@ def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
     return df
 
 def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
-    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+    available_cols = [col for col in byte_cols if col in df.columns]
 
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
@@ -70,7 +70,8 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
     else:
         groups = []
 
-    def _process_group(sub):
+    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    for gi, sub in enumerate(groups):
         mask = ~np.isnan(sub).any(axis=1)
         sub = sub[mask]
         if len(sub) > 1:
@@ -80,15 +81,7 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
                 c = np.abs(np.corrcoef(sub, rowvar=False))
             np.nan_to_num(c, copy=False, nan=0.0)
             np.fill_diagonal(c, 0.0)
-            return c.max(axis=0)
-        return np.zeros(n_cols, dtype=np.float64)
-
-    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
-    if len(groups) > 0:
-        with ThreadPoolExecutor() as executor:
-            results = list(executor.map(_process_group, groups))
-            for i, res in enumerate(results):
-                out[i] = res
+            out[gi] = c.max(axis=0)
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/entropy.py

@@ -4,7 +4,6 @@
 
 import argparse
 from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -37,7 +36,8 @@ def _entropy_col(a: np.ndarray) -> float:
     return float(-np.sum(p * np.log2(p)))
 
 def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
-    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+    available_cols = byte_cols
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
 
@@ -64,18 +64,10 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     else:
         groups = []
 
-    def _process_group(sub):
-        res = np.zeros(n_cols, dtype=np.float64)
-        for ci in range(n_cols):
-            res[ci] = _entropy_col(sub[:, ci])
-        return res
-
     out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
-    if len(groups) > 0:
-        with ThreadPoolExecutor() as executor:
-            results = list(executor.map(_process_group, groups))
-            for i, res in enumerate(results):
-                out[i] = res
+    for gi, sub in enumerate(groups):
+        for ci in range(n_cols):
+            out[gi, ci] = _entropy_col(sub[:, ci])
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/frequency.py

@@ -18,6 +18,7 @@ def _format_can_id_vec(s: pd.Series) -> pd.Series:
 def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
+    df['Formatted_ID'] = formatted
 
     counts = formatted.value_counts()
     freq_df = pd.DataFrame({

stats/id_viewer.py

@@ -7,7 +7,7 @@ from pathlib import Path
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
-from plotly_resampler import FigureResampler
+import lttbc
 from stats.utils.extractor import load_data
 
 def _format_can_id_vec(s: pd.Series) -> pd.Series:
@@ -41,24 +41,29 @@ def prepare_data(df, target_id):
     return filtered, byte_cols
 
 def plot_bits(df, byte_cols, can_id, title):
-    fig = FigureResampler(
-        resampled_trace_prefix_suffix=("", ""),
-        show_mean_aggregation_size=False
-    )
+    fig = go.Figure()
     colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf']
     n = len(byte_cols)
+    max_points = 2000
 
     x = df['Timestamp'].to_numpy() if not df.empty else np.array([])
     for i, col in enumerate(byte_cols):
         y = df[col].to_numpy(dtype=np.float32, copy=False) if not df.empty else np.array([])
 
-        fig.add_trace(go.Scatter(
+        if len(x) > max_points and len(x) == len(y):
+            x_plot, y_plot = lttbc.downsample(x, y, max_points)
+        else:
+            x_plot, y_plot = x, y
+
+        fig.add_trace(go.Scattergl(
+            x=x_plot,
+            y=y_plot,
             mode='lines',
             line=dict(shape='hv', width=2, color=colors[i % len(colors)]),
             name=col.upper(),
             legendgroup=col.upper(),
             hovertemplate=f"<b>{col.upper()}</b><br>Time: %{{x}}<br>Value: %{{y}}<extra></extra>",
-        ), hf_x=x, hf_y=y)
+        ))
 
     all_button = dict(label='ALL', method='restyle', args=[{'visible': [True] * n}])
     none_button = dict(label='NONE', method='restyle', args=[{'visible': ['legendonly'] * n}])