Merge pull request 'Implement multi-vehicle support and CAN log viewing options' (#7) from dev-dash into main

Reviewed-on: erickahmed/CANveyor#7

.python-version

@@ -0,0 +1 @@
+3.14

COPYRIGHT.md

@@ -0,0 +1,9 @@
+# Copyright
+
+Copyright © 2026 Erick Ahmed
+
+The source code in this repository is licensed under the **GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later)**.
+
+A copy of the license is provided in the `LICENSE` file. If any discrepancy exists between this notice and the `LICENSE` file, the `LICENSE` file shall prevail.
+
+Any third-party components included in this repository at any point during developement remain the property of their respective copyright holders and are subject to their own license terms.

LICENSE

@@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found.
 Copyright (C) <year>  <name of author>
 
 This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
+it under the terms of the GNU Affero General Public License as published
-    the Free Software Foundation, either version 3 of the License, or
+by the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.
 
 This program is distributed in the hope that it will be useful,

decoder.py

@@ -0,0 +1,86 @@
+# File: decoder.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import argparse
+import polars as pl
+
+def get_j1939_mask() -> pl.Expr:
+    """
+    Returns a Polars expression representing the strict J1939 filtering rules.
+    """
+    id_int = pl.col("ID").str.to_integer(base=16).cast(pl.UInt32)
+    return id_int > 0x7FF
+
+def decode_j1939_metadata(lf: pl.LazyFrame) -> pl.LazyFrame:
+    """
+    Decodes J1939 fields and bundles them into a Struct column.
+    """
+    id_int = pl.col("ID").str.to_integer(base=16).cast(pl.UInt32)
+
+    priority = ((id_int // 67108864) % 8).cast(pl.UInt8)
+    pf = ((id_int // 65536) % 256).cast(pl.UInt8)
+    ps = ((id_int // 256) % 256).cast(pl.UInt8)
+    sa = (id_int % 256).cast(pl.UInt8)
+
+    da = pl.when(pf < 240).then(ps).otherwise(pl.lit(255, dtype=pl.UInt8)).cast(pl.UInt8)
+
+    pgn = pl.when(pf < 240).then(
+        ((id_int // 256) & 0x3FF00)
+    ).otherwise(
+        ((id_int // 256) & 0x3FFFF)
+    ).cast(pl.UInt32)
+
+    return lf.with_columns(
+        pl.struct([
+            priority.alias("Priority"),
+            pf.alias("PF"),
+            ps.alias("PS"),
+            sa.alias("SA"),
+            da.alias("DA"),
+            pgn.alias("PGN")
+        ]).alias("j1939_metadata")
+    )
+
+def decode_j1939_frames(df: pl.DataFrame) -> pl.DataFrame:
+    id_int = pl.col("ID").str.to_integer(base=16).cast(pl.UInt32)
+
+    is_j1939 = id_int > 0x7FF
+
+    priority = ((id_int // 67108864) % 8).cast(pl.UInt8)
+    pf = ((id_int // 65536) % 256).cast(pl.UInt8)
+    ps = ((id_int // 256) % 256).cast(pl.UInt8)
+    sa = (id_int % 256).cast(pl.UInt8)
+
+    da = pl.when(pf < 240).then(ps).otherwise(pl.lit(255, dtype=pl.UInt8)).cast(pl.UInt8)
+
+    pgn = pl.when(pf < 240).then(
+        ((id_int // 256) & 0x3FF00)
+    ).otherwise(
+        ((id_int // 256) & 0x3FFFF)
+    ).cast(pl.UInt32)
+
+    j1939_meta = pl.when(is_j1939).then(
+        pl.struct([
+            priority.alias("Priority"),
+            pf.alias("PF"),
+            ps.alias("PS"),
+            sa.alias("SA"),
+            da.alias("DA"),
+            pgn.alias("PGN")
+        ])
+    ).otherwise(None)
+
+    return df.with_columns(j1939_meta.alias("j1939_metadata"))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="J1939 decoder")
+    parser.add_argument("input_parquet", help="Path to the raw .parquet file")
+    parser.add_argument("output_parquet", help="Path to save the decoded .parquet file")
+    args = parser.parse_args()
+
+    df = pl.scan_parquet(args.input_parquet).collect()
+    decoded_df = decode_j1939_frames(df)
+
+    decoded_df.write_parquet(args.output_parquet)

logs/view.py

@@ -0,0 +1,83 @@
+# File: logs/view.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import pandas as pd
+from dash import html, dash_table, dcc
+import dash_bootstrap_components as dbc
+
+PAGE_SIZE = 25000
+
+def prepare_logs_data(df: pd.DataFrame) -> pd.DataFrame:
+    if df is None or df.empty:
+        return pd.DataFrame()
+
+    df = df.copy()
+
+    if 'j1939_metadata' in df.columns:
+        df['Priority'] = df['j1939_metadata'].apply(lambda x: x.get('Priority') if isinstance(x, dict) else None)
+        df['PF'] = df['j1939_metadata'].apply(lambda x: x.get('PF') if isinstance(x, dict) else None)
+        df['PS'] = df['j1939_metadata'].apply(lambda x: x.get('PS') if isinstance(x, dict) else None)
+        df['SA'] = df['j1939_metadata'].apply(lambda x: x.get('SA') if isinstance(x, dict) else None)
+        df['DA'] = df['j1939_metadata'].apply(lambda x: x.get('DA') if isinstance(x, dict) else None)
+        df['PGN'] = df['j1939_metadata'].apply(lambda x: x.get('PGN') if isinstance(x, dict) else None)
+    else:
+        for col in ['Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']:
+            df[col] = None
+
+    for i in range(8):
+        col = f'b{i}'
+        if col in df.columns:
+            df[col] = df[col].apply(lambda x: f"{int(x):02X}" if pd.notna(x) else "")
+        else:
+            df[col] = ""
+
+    if 'ID' in df.columns:
+        df['ID'] = df['ID'].astype(str)
+
+    display_cols = ['Timestamp', 'ID', 'DLC', 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6', 'b7', 'Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']
+    display_df = df[[c for c in display_cols if c in df.columns]]
+
+    return display_df.fillna("")
+
+def get_logs_table_component():
+    return html.Div([
+        html.Div(id='logs-info-text', className="text-muted mb-2"),
+        dash_table.DataTable(
+            id='logs-table',
+            virtualization=True,
+            page_action='none',
+            style_table={'overflowX': 'auto', 'height': '70vh', 'overflowY': 'auto'},
+            style_header={
+                'backgroundColor': '#1a1a1a',
+                'color': 'white',
+                'fontWeight': 'bold',
+                'textAlign': 'center',
+                'position': 'sticky',
+                'top': 0
+            },
+            style_data={
+                'backgroundColor': '#f8f9fa',
+                'color': '#2a2a2a',
+                'textAlign': 'center'
+            },
+            style_data_conditional=[
+                {
+                    'if': {'row_index': 'odd'},
+                    'backgroundColor': 'rgb(240, 240, 240)'
+                }
+            ],
+            style_cell={
+                'minWidth': '80px',
+                'padding': '5px',
+                'textAlign': 'center',
+                'fontFamily': 'Segoe UI, Arial, sans-serif'
+            }
+        ),
+        html.Div([
+            dbc.Button("Prev", id='logs-prev-btn', color="secondary", outline=True, size="sm", className="me-2"),
+            html.Div(id='logs-page-nav', className="d-inline-block", style={'verticalAlign': 'middle'}),
+            dbc.Button("Next", id='logs-next-btn', color="secondary", outline=True, size="sm", className="ms-2"),
+        ], className="d-flex justify-content-center align-items-center mt-3"),
+        dcc.Store(id='logs-current-page', data=0),
+    ])

main.py

@@ -0,0 +1,395 @@
+# File: main.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import os
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+
+import polars as pl
+import dash
+from dash import dcc, html, Input, Output, State
+import dash_bootstrap_components as dbc
+import numpy as np
+
+from parser import parse_log, parse_csv
+from decoder import decode_j1939_frames
+from stats.utils.extractor import load_data
+from stats.id_viewer import _format_can_id_vec, plot_bits
+from stats.frequency import calculate_frequency, plot_frequency
+from stats.correlation import calculate_correlation, plot_correlation_heatmap
+from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap
+from logs.view import get_logs_table_component, prepare_logs_data
+
+RAW_LOG_DIR = "data/logs"
+PAGE_SIZE = 25000
+
+def parse_vehicle_from_filename(filename: str):
+    stem = Path(filename).stem
+    if '-' in stem:
+        brand, model_part = stem.split('-', 1)
+    else:
+        brand, model_part = stem, "Unknown"
+    model = model_part.replace('_', ' ')
+    vehicle = f"{brand} {model}".strip()
+    return vehicle, brand, model
+
+def run_pipeline():
+    os.makedirs(RAW_LOG_DIR, exist_ok=True)
+    os.makedirs("data/csv", exist_ok=True)
+    os.makedirs("data/parquet", exist_ok=True)
+
+    for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
+        vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
+
+        bus1_csv = f"data/csv/{vehicle}_bus1.csv"
+        bus2_csv = f"data/csv/{vehicle}_bus2.csv"
+        bus1_parquet = f"data/parquet/{vehicle}_bus1.parquet"
+        bus2_parquet = f"data/parquet/{vehicle}_bus2.parquet"
+        bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
+        bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
+
+        if not Path(bus1_decoded).exists() or not Path(bus2_decoded).exists():
+            print(f"Parsing raw log: {log_file.name}...")
+            parse_log(str(log_file), bus1_csv, bus2_csv)
+
+            print("Converting to parquet...")
+            parse_csv(bus1_csv).sink_parquet(bus1_parquet)
+            parse_csv(bus2_csv).sink_parquet(bus2_parquet)
+
+            print("Decoding J1939...")
+            df1 = pl.read_parquet(bus1_parquet)
+            df2 = pl.read_parquet(bus2_parquet)
+            dec1 = decode_j1939_frames(df1)
+            dec2 = decode_j1939_frames(df2)
+            dec1.write_parquet(bus1_decoded)
+            dec2.write_parquet(bus2_decoded)
+
+run_pipeline()
+
+print("Loading data into memory...")
+DATA = {}
+VEHICLE_META = {}
+
+for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
+    vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
+    VEHICLE_META[vehicle] = {"brand": brand, "model": model}
+
+    bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
+    bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
+
+    if Path(bus1_decoded).exists() and Path(bus2_decoded).exists():
+        DATA[vehicle] = {
+            "Bus 1": load_data(bus1_decoded),
+            "Bus 2": load_data(bus2_decoded)
+        }
+
+PRECOMPUTED_FIGURES = {}
+DATA_BY_ID = {}
+CORR_CACHE = {}
+PREPARED_LOGS_CACHE = {}
+
+def process_bus_data(vehicle, bus, df):
+    precomp = {}
+    precomp[f"{vehicle}_{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{vehicle} {bus} Frequency")
+    precomp[f"{vehicle}_{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{vehicle} {bus} Byte-Level Entropy")
+
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    formatted = _format_can_id_vec(df[can_id_col])
+    df = df.assign(Formatted_ID=formatted)
+    df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')
+
+    grouped = {}
+    for can_id, group in df.groupby(by='Formatted_ID'):
+        byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns]
+        if not group.empty and len(byte_cols) > 0:
+            arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False)
+            if len(arr) > 1:
+                changed = np.any(arr[1:] != arr[:-1], axis=1)
+                keep = np.concatenate(([True], changed))
+                group = group.iloc[keep]
+        grouped[can_id] = (group, byte_cols)
+
+    return vehicle, bus, precomp, grouped
+
+with ThreadPoolExecutor() as executor:
+    futures = []
+    for vehicle, buses in DATA.items():
+        for bus, df in buses.items():
+            futures.append(executor.submit(process_bus_data, vehicle, bus, df))
+    for future in futures:
+        v, b, precomp, grouped = future.result()
+        PRECOMPUTED_FIGURES.update(precomp)
+        DATA_BY_ID[(v, b)] = grouped
+
+app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+app.config.suppress_callback_exceptions = True
+
+app.layout = dbc.Container([
+    html.H1("CANveyor", className="my-4"),
+    dbc.Tabs([
+        dbc.Tab(label="Overview", tab_id="overview", children=[
+            html.Div(id="overview-content")
+        ]),
+        dbc.Tab(label="Logs", tab_id="logs", children=[
+            dbc.Row([
+                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='logs-vehicle-selector',
+                    options=[{'label': v, 'value': v} for v in DATA.keys()],
+                    value=list(DATA.keys())[0] if DATA else None,
+                    clearable=False
+                ), width=3, className="me-4"),
+                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='logs-bus-selector',
+                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
+                    value='Bus 1',
+                    clearable=False
+                ), width=2),
+            ], className="mb-3 mt-3", align="end"),
+            get_logs_table_component()
+        ]),
+        dbc.Tab(label="Statistics", tab_id="statistics", children=[
+            dbc.Row([
+                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='vehicle-selector',
+                    options=[{'label': v, 'value': v} for v in DATA.keys()],
+                    value=list(DATA.keys())[0] if DATA else None,
+                    clearable=False
+                ), width=3, className="me-4"),
+                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='bus-selector',
+                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
+                    value='Bus 1',
+                    clearable=False
+                ), width=2),
+            ], className="mb-3 mt-3", align="end"),
+            dbc.Tabs([
+                dbc.Tab(label="Frequency", tab_id="freq"),
+                dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
+                dbc.Tab(label="Correlation", tab_id="corr"),
+                dbc.Tab(label="Entropy", tab_id="entropy"),
+            ], id="tabs", active_tab="freq"),
+            html.Div(id="tab-content", className="mt-3")
+        ])
+    ], id="main-tabs", active_tab="statistics")
+], fluid=True)
+
+def get_prepared_logs(vehicle, bus):
+    cache_key = (vehicle, bus)
+    if cache_key not in PREPARED_LOGS_CACHE:
+        df = DATA[vehicle][bus]
+        PREPARED_LOGS_CACHE[cache_key] = prepare_logs_data(df)
+    return PREPARED_LOGS_CACHE[cache_key]
+
+def build_page_buttons(current_page: int, total_pages: int, max_buttons: int = 15) -> list:
+    buttons: list = []
+    if total_pages <= 1:
+        return buttons
+
+    half = max_buttons // 2
+    start = max(0, current_page - half)
+    end = min(total_pages, start + max_buttons)
+    if end - start < max_buttons:
+        start = max(0, end - max_buttons)
+
+    if start > 0:
+        buttons.append(
+            dbc.Button("1", id={'type': 'page-btn', 'index': 0}, color="secondary", outline=True, size="sm", className="me-1")
+        )
+        if start > 1:
+            buttons.append(html.Span("…", className="mx-1 align-middle"))
+
+    for i in range(start, end):
+        is_current = (i == current_page)
+        buttons.append(
+            dbc.Button(
+                str(i + 1),
+                id={'type': 'page-btn', 'index': i},
+                size="sm",
+                color="primary" if is_current else "secondary",
+                outline=not is_current,
+                className="me-1",
+                disabled=is_current,
+            )
+        )
+
+    if end < total_pages:
+        if end < total_pages - 1:
+            buttons.append(html.Span("…", className="mx-1 align-middle"))
+        buttons.append(
+            dbc.Button(
+                str(total_pages),
+                id={'type': 'page-btn', 'index': total_pages - 1},
+                color="secondary", outline=True, size="sm", className="me-1",
+            )
+        )
+
+    return buttons
+
+@app.callback(
+    Output('logs-table', 'data'),
+    Output('logs-table', 'columns'),
+    Output('logs-info-text', 'children'),
+    Output('logs-page-nav', 'children'),
+    Output('logs-current-page', 'data'),
+    Input('logs-vehicle-selector', 'value'),
+    Input('logs-bus-selector', 'value'),
+    Input('logs-prev-btn', 'n_clicks'),
+    Input('logs-next-btn', 'n_clicks'),
+    Input({'type': 'page-btn', 'index': dash.ALL}, 'n_clicks'),
+    State('logs-current-page', 'data'),
+)
+def update_logs_table(vehicle, bus, prev_clicks, next_clicks, page_btn_clicks, current_page):
+    if (not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]):
+        return [], [], "No data available", [], 0
+
+    prepared_df = get_prepared_logs(vehicle, bus)
+    total_rows = len(prepared_df)
+
+    if total_rows == 0:
+        return [], [], "No data available", [], 0
+
+    total_pages = max(1, (total_rows + PAGE_SIZE - 1) // PAGE_SIZE)
+
+    ctx = dash.callback_context
+    triggered_id = ctx.triggered_id
+
+    current_page = current_page if current_page is not None else 0
+
+    if triggered_id in ('logs-vehicle-selector', 'logs-bus-selector'):
+        current_page = 0
+
+    elif triggered_id == 'logs-prev-btn':
+        current_page = max(0, current_page - 1)
+
+    elif triggered_id == 'logs-next-btn':
+        current_page = current_page + 1
+
+    elif isinstance(triggered_id, dict) and triggered_id.get('type') == 'page-btn':
+        if ctx.triggered and ctx.triggered[0]['value']:
+            current_page = triggered_id['index']
+
+    current_page = max(0, min(current_page, total_pages - 1))
+
+    start_idx = current_page * PAGE_SIZE
+    end_idx = min(start_idx + PAGE_SIZE, total_rows)
+    page_data = prepared_df.iloc[start_idx:end_idx].to_dict('records')
+
+    columns = [{"name": i, "id": i} for i in prepared_df.columns]
+
+    info_text = (f"Page {current_page + 1} of {total_pages} | "
+                 f"Showing rows {start_idx + 1:,}–{end_idx:,} "
+                 f"of {total_rows:,} total frames")
+
+    page_buttons = build_page_buttons(current_page, total_pages)
+
+    return page_data, columns, info_text, page_buttons, current_page
+
+@app.callback(
+    Output('tab-content', 'children'),
+    Input('tabs', 'active_tab'),
+    Input('vehicle-selector', 'value'),
+    Input('bus-selector', 'value')
+)
+def render_content(tab, vehicle, bus):
+    if not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]:
+        return html.Div("No data available")
+
+    df = DATA[vehicle][bus]
+
+    if tab == 'freq':
+        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_freq"], style={'height': '80vh'})
+
+    elif tab == 'id_viewer':
+        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
+        return html.Div([
+            html.Label("Select CAN ID:"),
+            dcc.Dropdown(
+                id='id-selector',
+                options=[{'label': i, 'value': i} for i in ids],
+                value=ids[0] if ids else None,
+                clearable=False,
+                style={'width': '50%', 'marginBottom': '10px'}
+            ),
+            dcc.Graph(id='id-viewer-graph', style={'height': '70vh'})
+        ])
+
+    elif tab == 'corr':
+        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
+        return html.Div([
+            dbc.Row([
+                dbc.Col(html.Label("Method:"), width=1, className="mt-2"),
+                dbc.Col(dcc.Dropdown(
+                    id='corr-method',
+                    options=[{'label': 'Pearson', 'value': 'pearson'}, {'label': 'Spearman', 'value': 'spearman'}],
+                    value='pearson',
+                    clearable=False
+                ), width=2),
+                dbc.Col(html.Label("Target ID:"), width=1, className="mt-2"),
+                dbc.Col(dcc.Dropdown(
+                    id='corr-target',
+                    options=[{'label': 'All IDs (Max Corr)', 'value': 'all'}] + [{'label': i, 'value': i} for i in ids],
+                    value='all',
+                    clearable=True
+                ), width=4),
+            ], className="mb-3"),
+            dcc.Graph(id='corr-graph', style={'height': '80vh'})
+        ])
+
+    elif tab == 'entropy':
+        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_entropy"], style={'height': '80vh'})
+
+    return html.Div("Tab not found")
+
+@app.callback(
+    Output('id-viewer-graph', 'figure'),
+    Input('id-selector', 'value'),
+    Input('vehicle-selector', 'value'),
+    Input('bus-selector', 'value'),
+    Input('tabs', 'active_tab'),
+)
+def update_id_viewer(selected_id, vehicle, bus, tab):
+    if tab != 'id_viewer' or not selected_id or not vehicle or not bus:
+        return dash.no_update
+
+    grouped_data = DATA_BY_ID.get((vehicle, bus), {})
+    if selected_id not in grouped_data:
+        return dash.no_update
+
+    filtered_df, byte_cols = grouped_data[selected_id]
+    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{vehicle} {bus} Byte Visualization")
+
+@app.callback(
+    Output('corr-graph', 'figure'),
+    Input('corr-method', 'value'),
+    Input('corr-target', 'value'),
+    Input('vehicle-selector', 'value'),
+    Input('bus-selector', 'value'),
+    Input('tabs', 'active_tab'),
+)
+def update_corr(method, target, vehicle, bus, tab):
+    if tab != 'corr' or not vehicle or not bus:
+        return dash.no_update
+
+    target_id = None if target == 'all' or not target else target
+    cache_key = (vehicle, bus, method, target_id)
+
+    if cache_key not in CORR_CACHE:
+        df = DATA[vehicle][bus]
+        corr_df = calculate_correlation(df, method=method, target_id=target_id)
+        CORR_CACHE[cache_key] = corr_df
+    else:
+        corr_df = CORR_CACHE[cache_key]
+
+    title = f"{vehicle} {bus} Correlation"
+    if target_id:
+        title += f" ({target_id})"
+
+    return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
+
+if __name__ == '__main__':
+    app.run(debug=False)

parser.py

@@ -1,21 +1,25 @@
+# File: parser.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 import re
 import csv
+import polars as pl
 from pathlib import Path
+from typing import Union
 
-def parse_can_log(input_path: str | Path, out_bus1: str | Path, out_bus2: str | Path) -> None:
+PathLike = Union[str, Path]
+
+def parse_log(input_path: PathLike, out_bus1: PathLike, out_bus2: PathLike) -> None:
     """
-    Parses a CAN bus log file from CANdigger and saves valid frames to separate CSV files for Bus 1 and Bus 2.
+    Parses a raw CAN bus log from CANdigger using regex and saves valid frames to separate CSV.
     Corrupted, incomplete, or debug frames are silently discarded.
     """
     input_file = Path(input_path)
     out1_file = Path(out_bus1)
     out2_file = Path(out_bus2)
 
-    # Group 1: Bus (C1 or C2)
+    start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{7,8})\s+([0-9A-Fa-f]{1,2})\s+')
-    # Group 2: ID (1 to 8 hex chars)
-    # Group 3: DLC (1 to 2 hex chars)
-    start_pattern = re.compile(r'(C[12]):([0-9A-Fa-f]{1,8})\s+([0-9A-Fa-f]{1,2})\s+')
-
     byte_pattern = re.compile(r'^[0-9A-Fa-f]{2}$')
 
     with input_file.open('r', encoding='utf-8') as f_in, \
@@ -31,7 +35,12 @@ def parse_can_log(input_path: str | Path, out_bus1: str | Path, out_bus2: str |
         for line in f_in:
             for match in start_pattern.finditer(line):
                 bus = match.group(1)
-                can_id = match.group(2).upper()
+
+                can_id = match.group(2).upper().zfill(8)
+
+                if int(can_id, 16) > 0x1FFFFFFF:
+                    continue
+
                 dlc_str = match.group(3)
 
                 try:
@@ -61,21 +70,54 @@ def parse_can_log(input_path: str | Path, out_bus1: str | Path, out_bus2: str |
                     elif bus == 'C2':
                         writer2.writerow(row)
 
-if __name__ == '__main__':
+def parse_csv(csv_path: PathLike) -> pl.LazyFrame:
-    # Example usage:
+    """
-    # parse_can_log('can_traffic.txt', 'bus1_output.csv', 'bus2_output.csv')
+    Ingests a parsed CSV file, unpacks hex strings into 8 hex columns,
-    # or on CLI:
+    generates sequential timestamps if missing, and returns a Polars LazyFrame.
-    # python3 can_parser.py can_traffic.txt' bus1_output.csv bus2_output.csv
+    """
+    lf = pl.scan_csv(csv_path, schema_overrides={"ID": pl.String, "Data": pl.String})
+
+    if "Timestamp" not in lf.collect_schema().names():
+        lf = lf.with_row_index("Timestamp")
+
+    byte_exprs = []
+    for i in range(8):
+        expr = (
+            pl.col("Data").str.strip_chars().str.split(" ")
+            .list.get(i, null_on_oob=True)
+            .alias(f"b{i}")
+        )
+        byte_exprs.append(expr)
+
+    lf = lf.with_columns(byte_exprs).drop("Data")
+
+    return lf.with_columns([
+        pl.col("DLC").cast(pl.UInt8),
+        pl.col("Timestamp").cast(pl.Float64)
+    ])
 
 if __name__ == '__main__':
     import argparse
 
-        parser = argparse.ArgumentParser(description="Parse CAN bus logs to separate CSV files")
+    parser = argparse.ArgumentParser(description="CAN Bus Data Engine & Parser")
-        parser.add_argument("input", help="Path to the input .txt log file")
+    subparsers = parser.add_subparsers(dest="command", required=True, help="Available commands")
-        parser.add_argument("out_bus1", help="Output CSV filename for Bus 1 (C1)")
+
-        parser.add_argument("out_bus2", help="Output CSV filename for Bus 2 (C2)")
+    parser_csv = subparsers.add_parser("csv", help="Parse raw text log into Bus 1 and Bus 2 CSVs")
+    parser_csv.add_argument("input", help="Path to the .txt log file from CANdigger")
+    parser_csv.add_argument("out_bus1", help="Output CSV filename for Bus 1 (C1)")
+    parser_csv.add_argument("out_bus2", help="Output CSV filename for Bus 2 (C2)")
+
+    parser_parquet = subparsers.add_parser("parquet", help="Convert a parsed CSV into an optimized Parquet file")
+    parser_parquet.add_argument("input_csv", help="Path to the input .csv file")
+    parser_parquet.add_argument("output_parquet", help="Path to the output .parquet file")
 
     args = parser.parse_args()
-        parse_can_log(args.input, args.out_bus1, args.out_bus2)
 
-    pass
+    if args.command == "csv":
+        parse_log(args.input, args.out_bus1, args.out_bus2)
+        print(f"[+] Saved csv file to {args.out_bus1} and {args.out_bus2}")
+
+    elif args.command == "parquet":
+        print(f"[*] Processing {args.input_csv}...")
+        lf = parse_csv(args.input_csv)
+        lf.sink_parquet(args.output_parquet)

pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "CANveyor"
+version = "0.1.0"
+description = "J1939 CAN bus parser that works in pair with CANdigger"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "polars",
+    "dash",
+    "dash-bootstrap-components",
+    "numpy",
+    "pandas",
+    "plotly",
+    "plotly-resampler"
+]

stats/correlation.py

@@ -0,0 +1,178 @@
+# File: correlation.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import argparse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+
+from stats.utils.extractor import load_data
+from stats.utils.extractor import to_int
+
+def _format_can_id_vec(s: pd.Series) -> pd.Series:
+    s = s.astype('string').str.strip()
+    s = s.str.replace(r'^0x', '', case=False, regex=True)
+    s = s.str.upper()
+    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+
+def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
+    needs = [c for c in cols if not pd.api.types.is_numeric_dtype(df[c])]
+    if needs:
+        df = df.copy()
+        for c in needs:
+            df[c] = df[c].apply(to_int)
+    return df
+
+def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+
+    if not available_cols:
+        raise ValueError("No byte columns (b0-b7) found in the DataFrame")
+
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    identifiers = _format_can_id_vec(df[can_id_col]).to_numpy()
+
+    df_bytes = _ensure_int_bytes(df, available_cols)[available_cols]
+    data = df_bytes.to_numpy(dtype=np.float64, copy=False)
+
+    if target_id is not None:
+        target_id = _format_can_id_vec(pd.Series([target_id])).iloc[0]
+        mask = identifiers == target_id
+        if not mask.any():
+            raise ValueError(f"Identifier '{target_id}' not found in data")
+        sub = data[mask]
+        mask = ~np.isnan(sub).any(axis=1)
+        sub = sub[mask]
+        if method == 'spearman' and sub.shape[0] > 1:
+            sub = pd.DataFrame(sub).rank().to_numpy()
+        if sub.shape[0] > 1:
+            with np.errstate(divide='ignore', invalid='ignore'):
+                c = np.corrcoef(sub, rowvar=False)
+            np.nan_to_num(c, copy=False, nan=0.0)
+        else:
+            c = np.zeros((len(available_cols), len(available_cols)))
+        return pd.DataFrame(c, index=available_cols, columns=available_cols)
+
+    unique_ids, inverse = np.unique(identifiers, return_inverse=True)
+    n_cols = len(available_cols)
+
+    sort_idx = np.argsort(inverse, kind='stable')
+    data_sorted = data[sort_idx]
+    inverse_sorted = inverse[sort_idx]
+
+    if len(inverse_sorted) > 0:
+        split_points = np.flatnonzero(np.diff(inverse_sorted)) + 1
+        groups = np.split(data_sorted, split_points)
+    else:
+        groups = []
+
+    def _process_group(sub):
+        mask = ~np.isnan(sub).any(axis=1)
+        sub = sub[mask]
+        if len(sub) > 1:
+            if method == 'spearman':
+                sub = pd.DataFrame(sub).rank().to_numpy()
+            with np.errstate(divide='ignore', invalid='ignore'):
+                c = np.abs(np.corrcoef(sub, rowvar=False))
+            np.nan_to_num(c, copy=False, nan=0.0)
+            np.fill_diagonal(c, 0.0)
+            return c.max(axis=0)
+        return np.zeros(n_cols, dtype=np.float64)
+
+    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    if len(groups) > 0:
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(_process_group, groups))
+            for i, res in enumerate(results):
+                out[i] = res
+
+    result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
+    result.index.name = 'Identifier'
+    return result
+
+
+def plot_correlation_heatmap(corr_df: pd.DataFrame, target_id: str | None, title: str) -> go.Figure:
+    is_8x8 = target_id is not None
+
+    x = corr_df.columns.tolist()
+    y = corr_df.index.tolist()
+    z = corr_df.values
+
+    if is_8x8:
+        z_min, z_max = -1.0, 1.0
+        colorscale = [[0.0, "#2c7bb6"], [0.25, "#abd9e9"], [0.5, "#ffffff"], [0.75, "#fdae61"], [1.0, "#d7191c"]]
+        hover_template = "<b>%{y}</b> vs <b>%{x}</b><br>Correlation: %{z:.2f}<extra></extra>"
+    else:
+        z_min, z_max = 0.0, 1.0
+        colorscale = [[0.0, "#ffffff"], [0.2, "#fff5f0"], [0.4, "#fecc5c"], [0.6, "#fd8d3c"], [0.8, "#e31a1c"], [1.0, "#800026"]]
+        hover_template = "<b>%{y}</b><br>Byte %{x} max correlation: %{z:.2f}<extra></extra>"
+
+    fig = go.Figure(
+        data=go.Heatmap(
+            z=z, x=x, y=y,
+            zmin=z_min, zmax=z_max,
+            colorscale=colorscale,
+            xgap=3, ygap=3,
+            text=np.round(z, 2),
+            texttemplate="%{text}",
+            textfont={"size": 11, "color": "#2a2a2a", "family": "Segoe UI, Arial, sans-serif"},
+            hoverongaps=False,
+            hovertemplate=hover_template,
+            colorbar=dict(
+                title=dict(text="Correlation", side="top", font=dict(size=13, color="#1a1a1a")),
+                orientation="h", thickness=15, len=0.35,
+                x=1.0, xanchor="right", y=1.02, yanchor="bottom",
+                tickfont=dict(size=11, color="#2a2a2a"),
+                tickformat=".1f", outlinewidth=0.5, outlinecolor="#cccccc",
+            ),
+        )
+    )
+
+    fig.update_layout(
+        title=dict(text=title, font=dict(size=20, color="#1a1a1a"), x=0.5, xanchor="center", pad=dict(b=20)),
+        height=600 if is_8x8 else max(600, len(y) * 28 + 150),
+        autosize=True,
+        template="plotly_white",
+        xaxis=dict(
+            title=dict(text="Byte Position", font=dict(size=13, color="#1a1a1a")),
+            side="bottom" if is_8x8 else "top",
+            dtick=1, showgrid=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"), ticks="outside", ticklen=4, tickcolor="#cccccc",
+        ),
+        yaxis=dict(
+            title=dict(text="Byte Position" if is_8x8 else "PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
+            autorange="reversed", showgrid=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"), ticks="outside", ticklen=4, tickcolor="#cccccc", automargin=True,
+        ),
+        font=dict(family="Segoe UI, Arial, sans-serif", size=12, color="#2a2a2a"),
+        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI", bordercolor="#cccccc"),
+        margin=dict(l=200, r=40, t=120, b=60),
+    )
+    return fig
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Analyze CAN bus inter-byte correlation")
+    parser.add_argument("method", choices=["pearson", "spearman"], help="Correlation method to use")
+    parser.add_argument("input", type=Path, help="Path to the input CAN log file")
+    parser.add_argument("output", type=Path, nargs="?", default=Path("correlation_report.html"))
+    parser.add_argument("title", nargs="?", default="CAN Bus Inter-Byte Correlation")
+    parser.add_argument("--identifier", type=str, default=None)
+    args = parser.parse_args()
+
+    df = load_data(args.input)
+    corr_df = calculate_correlation(df, method=args.method, target_id=args.identifier)
+    display_title = f"{args.title} ({args.identifier})" if args.identifier else args.title
+    fig = plot_correlation_heatmap(corr_df, target_id=args.identifier, title=display_title)
+    config = {
+        "responsive": True,
+        "displaylogo": False,
+        "scrollZoom": True,
+        "modeBarButtonsToAdd": ["toggleSpikelines"],
+        "toImageButtonOptions": {"format": "png", "scale": 2},
+    }
+    fig.write_html(str(args.output), include_plotlyjs="cdn", config=config)

stats/entropy.py

@@ -0,0 +1,156 @@
+# File: entropy.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import argparse
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+
+from stats.utils.extractor import load_data
+from stats.utils.extractor import to_int
+
+def _format_can_id_vec(s: pd.Series) -> pd.Series:
+    s = s.astype('string').str.strip()
+    s = s.str.replace(r'^0x', '', case=False, regex=True)
+    s = s.str.upper()
+    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+
+def _entropy_col(a: np.ndarray) -> float:
+    a = a[~np.isnan(a)]
+    if a.size == 0:
+        return 0.0
+    a = a.astype(np.int64)
+    lo, hi = a.min(), a.max()
+    span = hi - lo + 1
+    if span <= 0:
+        return 0.0
+    if span > 1 << 20:
+        _, counts = np.unique(a, return_counts=True)
+    else:
+        counts = np.bincount(a - lo, minlength=span)
+        counts = counts[counts > 0]
+    p = counts / counts.sum()
+    return float(-np.sum(p * np.log2(p)))
+
+def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
+    if not available_cols:
+        raise ValueError("No byte columns (b0-b7) found in the DataFrame")
+
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    identifiers = _format_can_id_vec(df[can_id_col]).to_numpy()
+
+    needs = [c for c in available_cols if not pd.api.types.is_numeric_dtype(df[c])]
+    if needs:
+        df = df.copy()
+        for c in needs:
+            df[c] = df[c].apply(to_int)
+
+    data = df[available_cols].to_numpy(dtype=np.float64, copy=False)
+    unique_ids, inverse = np.unique(identifiers, return_inverse=True)
+    n_cols = len(available_cols)
+
+    sort_idx = np.argsort(inverse, kind='stable')
+    data_sorted = data[sort_idx]
+    inverse_sorted = inverse[sort_idx]
+
+    if len(inverse_sorted) > 0:
+        split_points = np.flatnonzero(np.diff(inverse_sorted)) + 1
+        groups = np.split(data_sorted, split_points)
+    else:
+        groups = []
+
+    def _process_group(sub):
+        res = np.zeros(n_cols, dtype=np.float64)
+        for ci in range(n_cols):
+            res[ci] = _entropy_col(sub[:, ci])
+        return res
+
+    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    if len(groups) > 0:
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(_process_group, groups))
+            for i, res in enumerate(results):
+                out[i] = res
+
+    result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
+    result.index.name = 'Identifier'
+    return result
+
+
+def plot_entropy_heatmap(entropy_df: pd.DataFrame, title: str) -> go.Figure:
+    x = entropy_df.columns.tolist()
+    y = entropy_df.index.tolist()
+    z = entropy_df.values
+
+    fig = go.Figure(
+        data=go.Heatmap(
+            z=z, x=x, y=y,
+            colorscale=[
+                [0.0, "#ffffff"],
+                [0.15, "#fff7ec"],
+                [0.35, "#fee8c8"],
+                [0.55, "#fdd49e"],
+                [0.75, "#fdbb84"],
+                [1.0, "#ef6548"],
+            ],
+            xgap=3, ygap=3,
+            text=np.round(z, 2),
+            texttemplate="%{text}",
+            textfont={"size": 11, "color": "#2a2a2a", "family": "Segoe UI, Arial, sans-serif"},
+            hoverongaps=False,
+            hovertemplate="<b>%{y}</b><br>Byte %{x}: %{z:.2f} bits<extra></extra>",
+            colorbar=dict(
+                title=dict(text="Entropy (bits)", side="top", font=dict(size=13, color="#1a1a1a")),
+                orientation="h", thickness=15, len=0.35,
+                x=1.0, xanchor="right", y=1.02, yanchor="bottom",
+                tickfont=dict(size=11, color="#2a2a2a"),
+                tickformat=".1f", outlinewidth=0.5, outlinecolor="#cccccc",
+            ),
+        )
+    )
+
+    fig.update_layout(
+        title=dict(text=title, font=dict(size=20, color="#1a1a1a"), x=0.5, xanchor="center", pad=dict(b=20)),
+        height=max(600, len(y) * 28 + 150),
+        autosize=True,
+        template="plotly_white",
+        xaxis=dict(
+            title=dict(text="Byte Position", font=dict(size=13, color="#1a1a1a")),
+            side="top", dtick=1, showgrid=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"), ticks="outside", ticklen=4, tickcolor="#cccccc",
+        ),
+        yaxis=dict(
+            title=dict(text="PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
+            autorange="reversed", showgrid=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"), ticks="outside", ticklen=4, tickcolor="#cccccc", automargin=True,
+        ),
+        font=dict(family="Segoe UI, Arial, sans-serif", size=12, color="#2a2a2a"),
+        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI", bordercolor="#cccccc"),
+        margin=dict(l=200, r=40, t=120, b=60),
+    )
+    return fig
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Analyze CAN bus byte-level entropy")
+    parser.add_argument("input", type=Path, help="Path to the input CAN log file")
+    parser.add_argument("output", type=Path, nargs="?", default=Path("entropy_report.html"))
+    parser.add_argument("title", nargs="?", default="CAN Bus Byte-Level Entropy")
+    args = parser.parse_args()
+
+    df = load_data(args.input)
+    entropy_df = calculate_byte_entropy(df)
+    fig = plot_entropy_heatmap(entropy_df, title=args.title)
+    config = {
+        "responsive": True,
+        "displaylogo": False,
+        "scrollZoom": True,
+        "modeBarButtonsToAdd": ["toggleSpikelines"],
+        "toImageButtonOptions": {"format": "png", "scale": 2},
+    }
+    fig.write_html(str(args.output), include_plotlyjs="cdn", config=config)

stats/frequency.py

@@ -0,0 +1,126 @@
+# File: frequency.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import argparse
+from pathlib import Path
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from stats.utils.extractor import load_data
+
+def _format_can_id_vec(s: pd.Series) -> pd.Series:
+    s = s.astype('string').str.strip()
+    s = s.str.replace(r'^0x', '', case=False, regex=True)
+    s = s.str.upper()
+    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+
+def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    formatted = _format_can_id_vec(df[can_id_col])
+
+    counts = formatted.value_counts()
+    freq_df = pd.DataFrame({
+        'Identifier': counts.index,
+        'Count': counts.to_numpy(),
+    })
+    total = counts.sum()
+    freq_df['Percentage'] = np.round(freq_df['Count'] / total * 100, 2) if total else 0.0
+    return freq_df.sort_values('Count', ascending=True).reset_index(drop=True)
+
+def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
+    n = len(stats_df)
+    fig = go.Figure(go.Bar(
+        y=stats_df['Identifier'],
+        x=stats_df['Count'],
+        orientation='h',
+        marker=dict(
+            color=stats_df['Count'],
+            colorscale='Turbo',
+            cmin=int(stats_df['Count'].min()) if n else 0,
+            cmax=int(stats_df['Count'].max()) if n else 1,
+            line_width=0,
+        ),
+        customdata=stats_df[['Percentage']].to_numpy(),
+        hovertemplate="<b>%{y}</b><br>Count: %{x:,}<br>Share: %{customdata[0]}%<extra></extra>",
+        texttemplate='%{x:,}',
+        textposition='outside',
+        cliponaxis=False,
+    ))
+
+    fig.update_layout(
+        height=max(600, n * 18),
+        autosize=True,
+        template='plotly_white',
+        xaxis=dict(
+            type='log',
+            title=dict(text="Message count [log scale]", font=dict(size=13, color="#1a1a1a")),
+            side="top",
+            dtick=1,
+            showgrid=False,
+            linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"),
+            ticks="outside",
+            ticklen=4,
+            tickcolor="#cccccc",
+        ),
+        yaxis=dict(
+            title=dict(text="PGN or CAN ID", font=dict(size=13, color="#1a1a1a")),
+            showgrid=False,
+            linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"),
+            ticks="outside",
+            ticklen=4,
+            tickcolor="#cccccc",
+            automargin=True,
+            type='category',
+        ),
+        font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
+        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI", bordercolor='#cccccc'),
+        margin=dict(l=200, r=40, t=120, b=60),
+        bargap=0.35,
+        coloraxis_colorbar=dict(
+            title=dict(text='Message Count', side='top'),
+            orientation='h',
+            thickness=15,
+            len=0.35,
+            x=1.0,
+            xanchor='right',
+            y=1.02,
+            yanchor='bottom',
+            tickformat=',',
+            outlinecolor='#cccccc',
+            outlinewidth=0.5,
+        ),
+        title=dict(text=title, font=dict(size=20, color='#1a1a1a'), x=0.5, xanchor='center', pad=dict(b=20)),
+    )
+    fig.update_xaxes(
+        showgrid=True, gridwidth=0.5, gridcolor='#e8e8e8',
+        zeroline=False, linecolor='#bdbdbd', mirror=False,
+        tickformat=',',
+        minor=dict(showgrid=True, gridcolor='#f4f4f4', gridwidth=0.5),
+    )
+    fig.update_yaxes(
+        showgrid=False, zeroline=False, linecolor='#bdbdbd',
+        ticks='outside', ticklen=4, tickcolor='#cccccc', automargin=True,
+    )
+    return fig
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Analyze CAN bus message frequency")
+    parser.add_argument("input", type=Path, help="Path to the input CAN log file")
+    parser.add_argument("output", type=Path, nargs="?", default=Path("freq_report.html"))
+    parser.add_argument("title", nargs="?", default="Frequency")
+    args = parser.parse_args()
+
+    df = load_data(args.input)
+    stats = calculate_frequency(df)
+    fig = plot_frequency(stats, title=args.title)
+    config = {
+        'responsive': True,
+        'displaylogo': False,
+        'scrollZoom': True,
+        'modeBarButtonsToAdd': ['toggleSpikelines'],
+        'toImageButtonOptions': {'format': 'png', 'scale': 2},
+    }
+    fig.write_html(str(args.output), include_plotlyjs='cdn', config=config)

stats/id_viewer.py

@@ -0,0 +1,143 @@
+# File: id_viewer.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import argparse
+from pathlib import Path
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from plotly_resampler import FigureResampler
+from stats.utils.extractor import load_data
+
+def _format_can_id_vec(s: pd.Series) -> pd.Series:
+    s = s.astype('string').str.strip()
+    s = s.str.replace(r'^0x', '', case=False, regex=True)
+    s = s.str.upper()
+    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+
+def prepare_data(df, target_id):
+    can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
+    formatted = _format_can_id_vec(df[can_id_col])
+    df = df.assign(Formatted_ID=formatted)
+    target_id_clean = _format_can_id_vec(pd.Series([target_id])).iloc[0]
+    filtered = df[df['Formatted_ID'] == target_id_clean]
+
+    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in filtered.columns]
+    if filtered.empty:
+        return filtered, byte_cols
+
+    for col in byte_cols:
+        if not pd.api.types.is_numeric_dtype(filtered[col]):
+            filtered = filtered.assign(**{col: pd.to_numeric(filtered[col], errors='coerce').astype('float32')})
+
+    filtered = filtered.sort_values('Timestamp', kind='stable')
+    arr = filtered[byte_cols].to_numpy(dtype=np.float32, copy=False)
+    if len(arr) > 1:
+        changed = np.any(arr[1:] != arr[:-1], axis=1)
+        keep = np.concatenate(([True], changed))
+        filtered = filtered.iloc[keep]
+
+    return filtered, byte_cols
+
+def plot_bits(df, byte_cols, can_id, title):
+    fig = FigureResampler(
+        resampled_trace_prefix_suffix=("", ""),
+        show_mean_aggregation_size=False
+    )
+    colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf']
+    n = len(byte_cols)
+
+    x = df['Timestamp'].to_numpy() if not df.empty else np.array([])
+    for i, col in enumerate(byte_cols):
+        y = df[col].to_numpy(dtype=np.float32, copy=False) if not df.empty else np.array([])
+
+        fig.add_trace(go.Scatter(
+            mode='lines',
+            line=dict(shape='hv', width=2, color=colors[i % len(colors)]),
+            name=col.upper(),
+            legendgroup=col.upper(),
+            hovertemplate=f"<b>{col.upper()}</b><br>Time: %{{x}}<br>Value: %{{y}}<extra></extra>",
+        ), hf_x=x, hf_y=y)
+
+    all_button = dict(label='ALL', method='restyle', args=[{'visible': [True] * n}])
+    none_button = dict(label='NONE', method='restyle', args=[{'visible': ['legendonly'] * n}])
+
+    fig.update_layout(
+        height=600,
+        autosize=True,
+        template='plotly_white',
+        title=dict(
+            text=f"{title} - ID: {can_id}",
+            font=dict(size=20, color='#1a1a1a'),
+            x=0.5, xanchor='center',
+            pad=dict(b=20),
+        ),
+        font=dict(family="Segoe UI, Arial, sans-serif", size=12, color='#2a2a2a'),
+        hoverlabel=dict(bgcolor="white", font_size=13, font_family="Segoe UI", bordercolor='#cccccc'),
+        margin=dict(l=60, r=40, t=120, b=140),
+        legend=dict(
+            orientation='h',
+            x=0.5, xanchor='center',
+            y=-0.18, yanchor='top',
+            title=None,
+            bgcolor='white',
+            bordercolor='#cccccc',
+            borderwidth=1,
+            font=dict(size=12, color="#2a2a2a"),
+            itemsizing='constant',
+            itemclick='toggle',
+            itemdoubleclick='toggleothers',
+        ),
+        xaxis=dict(
+            title=dict(text="Timestamp", font=dict(size=13, color="#1a1a1a")),
+            showgrid=True, gridwidth=0.5, gridcolor='#e8e8e8',
+            zeroline=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"),
+            ticks="outside", ticklen=4, tickcolor="#cccccc",
+            minor=dict(showgrid=True, gridcolor='#f4f4f4', gridwidth=0.5),
+        ),
+        yaxis=dict(
+            title=dict(text="Byte Value", font=dict(size=13, color="#1a1a1a")),
+            showgrid=True, gridwidth=0.5, gridcolor='#e8e8e8',
+            zeroline=False, linecolor="#bdbdbd",
+            tickfont=dict(size=12, color="#2a2a2a"),
+            ticks="outside", ticklen=4, tickcolor="#cccccc",
+        ),
+        updatemenus=[
+            dict(
+                type='buttons',
+                direction='right',
+                x=0.5, xanchor='center',
+                y=-0.06, yanchor='top',
+                buttons=[all_button, none_button],
+                bgcolor='white',
+                bordercolor='#cccccc',
+                borderwidth=1,
+                font=dict(size=11, color='#2a2a2a'),
+                pad=dict(l=5, r=5, t=5, b=5),
+            )
+        ],
+    )
+    return fig
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Visualize CAN bus byte changes over time")
+    parser.add_argument("input", type=Path, help="Path to the input CAN log file")
+    parser.add_argument("can_id", type=str, help="CAN ID to visualize")
+    parser.add_argument("output", type=Path, nargs="?", default=Path("bits_report.html"))
+    parser.add_argument("title", nargs="?", default="Byte Visualization")
+    args = parser.parse_args()
+
+    df = load_data(args.input)
+    filtered_df, byte_cols = prepare_data(df, args.can_id)
+    fig = plot_bits(filtered_df, byte_cols, args.can_id, title=args.title)
+
+    config = {
+        'responsive': True,
+        'displaylogo': False,
+        'scrollZoom': True,
+        'modeBarButtonsToAdd': ['toggleSpikelines'],
+        'toImageButtonOptions': {'format': 'png', 'scale': 2},
+    }
+    fig.write_html(str(args.output), include_plotlyjs='cdn', config=config)

stats/utils/extractor.py

@@ -0,0 +1,63 @@
+# File: extractor.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import json
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import polars as pl
+
+def to_int(x):
+    if isinstance(x, (int, np.integer)):
+        return int(x)
+    if isinstance(x, str):
+        try:
+            return int(x, 16)
+        except ValueError:
+            return np.nan
+    return np.nan
+
+def extract_id(row: pd.Series) -> str:
+    meta = row.get('j1939_metadata')
+    if pd.isna(meta):
+        return f"ID: {row['ID']}"
+    if isinstance(meta, str):
+        try:
+            meta = json.loads(meta)
+        except json.JSONDecodeError:
+            return f"ID: {row['ID']}"
+    if isinstance(meta, dict) and 'PGN' in meta:
+        return f"PGN: {meta['PGN']}"
+    return f"ID: {row['ID']}"
+
+def load_data(file_path: Path) -> pd.DataFrame:
+    lf = pl.scan_parquet(file_path)
+    schema = lf.collect_schema()
+    names = schema.names()
+
+    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in names]
+    if byte_cols:
+        lf = lf.with_columns([
+            pl.col(c).str.to_integer(base=16, strict=False).cast(pl.Int16).alias(c)
+            for c in byte_cols
+        ])
+
+    id_col = 'ID' if 'ID' in names else 'Identifier'
+    id_expr = pl.col(id_col).cast(pl.Utf8)
+
+    if 'j1939_metadata' in names:
+        try:
+            lf = lf.with_columns(
+                pl.when(pl.col('j1939_metadata').is_not_null())
+                .then(pl.lit('PGN: ') + pl.col('j1939_metadata').struct.field('PGN').cast(pl.Utf8))
+                .otherwise(pl.lit('ID: ') + id_expr)
+                .alias('Identifier')
+            )
+        except Exception:
+            lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
+    else:
+        lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
+
+    return lf.collect().to_pandas()