Refactor CANveyor dashboard architecture

- Replace `stats.utils.extractor` with dedicated `loader` and
  `converter`
  modules to improve code organization.
- Implement explicit pipeline stages for ingestion, decoding, and
  precomputation with caching.
- Standardize data loading and J1939 parsing logic across sub-modules.
- Enhance dashboard responsiveness by pre-calculating figures and
  downsampling ID-grouped data.
- Enforce strict typing and add docstrings to public components.

stats/correlation.py

@@ -1,25 +1,23 @@
-# File: correlation.py
+# File: stats/correlation.py
 # Copyright (C) 2026 Erick Ahmed
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+"""CAN bus inter-byte correlation analyzer and plotter."""
+
 import argparse
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
+from typing import List
 
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from stats.utils.extractor import load_data
-from stats.utils.extractor import to_int
+from stats.utils.converter import format_can_id_vec as _format_can_id_vec, to_int
+from stats.utils.loader import load_data
 
-def _format_can_id_vec(s: pd.Series) -> pd.Series:
-    s = s.astype('string').str.strip()
-    s = s.str.replace(r'^0x', '', case=False, regex=True)
-    s = s.str.upper()
-    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
 
-def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
+def _ensure_int_bytes(df: pd.DataFrame, cols: List[str]) -> pd.DataFrame:
     needs = [c for c in cols if not pd.api.types.is_numeric_dtype(df[c])]
     if needs:
         df = df.copy()
@@ -27,6 +25,7 @@ def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
             df[c] = df[c].apply(to_int)
     return df
 
+
 def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
     available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
 
@@ -70,7 +69,7 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
     else:
         groups = []
 
-    def _process_group(sub):
+    def _process_group(sub: np.ndarray) -> np.ndarray:
         mask = ~np.isnan(sub).any(axis=1)
         sub = sub[mask]
         if len(sub) > 1:

stats/entropy.py

@@ -1,23 +1,21 @@
-# File: entropy.py
+# File: stats/entropy.py
 # Copyright (C) 2026 Erick Ahmed
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+"""CAN bus byte-level entropy analyzer and plotter."""
+
 import argparse
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
+from typing import List
 
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 
-from stats.utils.extractor import load_data
-from stats.utils.extractor import to_int
+from stats.utils.converter import format_can_id_vec as _format_can_id_vec, to_int
+from stats.utils.loader import load_data
 
-def _format_can_id_vec(s: pd.Series) -> pd.Series:
-    s = s.astype('string').str.strip()
-    s = s.str.replace(r'^0x', '', case=False, regex=True)
-    s = s.str.upper()
-    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
 
 def _entropy_col(a: np.ndarray) -> float:
     a = a[~np.isnan(a)]
@@ -36,6 +34,7 @@ def _entropy_col(a: np.ndarray) -> float:
     p = counts / counts.sum()
     return float(-np.sum(p * np.log2(p)))
 
+
 def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
     if not available_cols:
@@ -64,7 +63,7 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     else:
         groups = []
 
-    def _process_group(sub):
+    def _process_group(sub: np.ndarray) -> np.ndarray:
         res = np.zeros(n_cols, dtype=np.float64)
         for ci in range(n_cols):
             res[ci] = _entropy_col(sub[:, ci])

stats/frequency.py

@@ -1,19 +1,19 @@
-# File: frequency.py
+# File: stats/frequency.py
 # Copyright (C) 2026 Erick Ahmed
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+"""CAN bus message frequency analyzer and plotter."""
+
 import argparse
 from pathlib import Path
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
-from stats.utils.extractor import load_data
 
-def _format_can_id_vec(s: pd.Series) -> pd.Series:
-    s = s.astype('string').str.strip()
-    s = s.str.replace(r'^0x', '', case=False, regex=True)
-    s = s.str.upper()
-    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+from stats.utils.converter import format_can_id_vec as _format_can_id_vec
+from stats.utils.loader import load_data
+
 
 def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
@@ -28,6 +28,7 @@ def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     freq_df['Percentage'] = np.round(freq_df['Count'] / total * 100, 2) if total else 0.0
     return freq_df.sort_values('Count', ascending=True).reset_index(drop=True)
 
+
 def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
     n = len(stats_df)
     fig = go.Figure(go.Bar(
@@ -106,6 +107,7 @@ def plot_frequency(stats_df: pd.DataFrame, title: str) -> go.Figure:
     )
     return fig
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Analyze CAN bus message frequency")
     parser.add_argument("input", type=Path, help="Path to the input CAN log file")

stats/id_viewer.py

@@ -1,25 +1,30 @@
-# File: id_viewer.py
+# File: stats/id_viewer.py
 # Copyright (C) 2026 Erick Ahmed
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+"""Interactive CAN bus byte-change visualizer."""
+
 import argparse
 from pathlib import Path
+from typing import List, Tuple
+
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from plotly_resampler import FigureResampler
-from stats.utils.extractor import load_data
 
-def _format_can_id_vec(s: pd.Series) -> pd.Series:
-    s = s.astype('string').str.strip()
-    s = s.str.replace(r'^0x', '', case=False, regex=True)
-    s = s.str.upper()
-    return s.fillna('UNKNOWN').replace('', 'UNKNOWN')
+from stats.utils.converter import format_can_id_vec as _format_can_id_vec
+from stats.utils.loader import load_data
 
-def prepare_data(df, target_id):
+_BYTE_COLORS = [
+    '#e41a1c', '#377eb8', '#4daf4a', '#984ea3',
+    '#ff7f00', '#ffff33', '#a65628', '#f781bf',
+]
+
+
+def prepare_data(df: pd.DataFrame, target_id: str) -> Tuple[pd.DataFrame, List[str]]:
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
-    formatted = _format_can_id_vec(df[can_id_col])
-    df = df.assign(Formatted_ID=formatted)
+    df = df.assign(Formatted_ID=_format_can_id_vec(df[can_id_col]))
     target_id_clean = _format_can_id_vec(pd.Series([target_id])).iloc[0]
     filtered = df[df['Formatted_ID'] == target_id_clean]
 
@@ -29,7 +34,9 @@ def prepare_data(df, target_id):
 
     for col in byte_cols:
         if not pd.api.types.is_numeric_dtype(filtered[col]):
-            filtered = filtered.assign(**{col: pd.to_numeric(filtered[col], errors='coerce').astype('float32')})
+            filtered = filtered.assign(
+                **{col: pd.to_numeric(filtered[col], errors='coerce').astype('float32')}
+            )
 
     filtered = filtered.sort_values('Timestamp', kind='stable')
     arr = filtered[byte_cols].to_numpy(dtype=np.float32, copy=False)
@@ -40,12 +47,12 @@ def prepare_data(df, target_id):
 
     return filtered, byte_cols
 
-def plot_bits(df, byte_cols, can_id, title):
+
+def plot_bits(df: pd.DataFrame, byte_cols: List[str], can_id: str, title: str) -> FigureResampler:
     fig = FigureResampler(
         resampled_trace_prefix_suffix=("", ""),
         show_mean_aggregation_size=False
     )
-    colors = ['#e41a1c', '#377eb8', '#4daf4a', '#984ea3', '#ff7f00', '#ffff33', '#a65628', '#f781bf']
     n = len(byte_cols)
 
     x = df['Timestamp'].to_numpy() if not df.empty else np.array([])
@@ -54,7 +61,7 @@ def plot_bits(df, byte_cols, can_id, title):
 
         fig.add_trace(go.Scatter(
             mode='lines',
-            line=dict(shape='hv', width=2, color=colors[i % len(colors)]),
+            line=dict(shape='hv', width=2, color=_BYTE_COLORS[i % len(_BYTE_COLORS)]),
             name=col.upper(),
             legendgroup=col.upper(),
             hovertemplate=f"<b>{col.upper()}</b><br>Time: %{{x}}<br>Value: %{{y}}<extra></extra>",
@@ -121,6 +128,7 @@ def plot_bits(df, byte_cols, can_id, title):
     )
     return fig
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Visualize CAN bus byte changes over time")
     parser.add_argument("input", type=Path, help="Path to the input CAN log file")

stats/utils/extractor.py

@@ -1,63 +0,0 @@
-# File: extractor.py
-# Copyright (C) 2026 Erick Ahmed
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
-import json
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import polars as pl
-
-def to_int(x):
-    if isinstance(x, (int, np.integer)):
-        return int(x)
-    if isinstance(x, str):
-        try:
-            return int(x, 16)
-        except ValueError:
-            return np.nan
-    return np.nan
-
-def extract_id(row: pd.Series) -> str:
-    meta = row.get('j1939_metadata')
-    if pd.isna(meta):
-        return f"ID: {row['ID']}"
-    if isinstance(meta, str):
-        try:
-            meta = json.loads(meta)
-        except json.JSONDecodeError:
-            return f"ID: {row['ID']}"
-    if isinstance(meta, dict) and 'PGN' in meta:
-        return f"PGN: {meta['PGN']}"
-    return f"ID: {row['ID']}"
-
-def load_data(file_path: Path) -> pd.DataFrame:
-    lf = pl.scan_parquet(file_path)
-    schema = lf.collect_schema()
-    names = schema.names()
-
-    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in names]
-    if byte_cols:
-        lf = lf.with_columns([
-            pl.col(c).str.to_integer(base=16, strict=False).cast(pl.Int16).alias(c)
-            for c in byte_cols
-        ])
-
-    id_col = 'ID' if 'ID' in names else 'Identifier'
-    id_expr = pl.col(id_col).cast(pl.Utf8)
-
-    if 'j1939_metadata' in names:
-        try:
-            lf = lf.with_columns(
-                pl.when(pl.col('j1939_metadata').is_not_null())
-                .then(pl.lit('PGN: ') + pl.col('j1939_metadata').struct.field('PGN').cast(pl.Utf8))
-                .otherwise(pl.lit('ID: ') + id_expr)
-                .alias('Identifier')
-            )
-        except Exception:
-            lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
-    else:
-        lf = lf.with_columns((pl.lit('ID: ') + id_expr).alias('Identifier'))
-
-    return lf.collect().to_pandas()