Implement infinite scroll for logs table

logs/view.py

@@ -0,0 +1,76 @@
+# File: logs/view.py
+# Copyright (C) 2026 Erick Ahmed
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import pandas as pd
+from dash import html, dash_table
+
+def prepare_logs_data(df: pd.DataFrame) -> pd.DataFrame:
+    if df is None or df.empty:
+        return pd.DataFrame()
+
+    df = df.copy()
+
+    if 'j1939_metadata' in df.columns:
+        df['Priority'] = df['j1939_metadata'].apply(lambda x: x.get('Priority') if isinstance(x, dict) else None)
+        df['PF'] = df['j1939_metadata'].apply(lambda x: x.get('PF') if isinstance(x, dict) else None)
+        df['PS'] = df['j1939_metadata'].apply(lambda x: x.get('PS') if isinstance(x, dict) else None)
+        df['SA'] = df['j1939_metadata'].apply(lambda x: x.get('SA') if isinstance(x, dict) else None)
+        df['DA'] = df['j1939_metadata'].apply(lambda x: x.get('DA') if isinstance(x, dict) else None)
+        df['PGN'] = df['j1939_metadata'].apply(lambda x: x.get('PGN') if isinstance(x, dict) else None)
+    else:
+        for col in ['Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']:
+            df[col] = None
+
+    for i in range(8):
+        col = f'b{i}'
+        if col in df.columns:
+            df[col] = df[col].apply(lambda x: f"{int(x):02X}" if pd.notna(x) else "")
+        else:
+            df[col] = ""
+
+    if 'ID' in df.columns:
+        df['ID'] = df['ID'].astype(str)
+
+    display_cols = ['Timestamp', 'ID', 'DLC', 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6', 'b7', 'Priority', 'PF', 'PS', 'SA', 'DA', 'PGN']
+    display_df = df[[c for c in display_cols if c in df.columns]]
+
+    return display_df.fillna("")
+
+def get_logs_table_component():
+    return html.Div([
+        html.Div(id='logs-info-text', className="text-muted mb-2"),
+        dash_table.DataTable(
+            id='logs-table',
+            virtualization=True,
+            page_action='none',
+            style_table={'overflowX': 'auto', 'height': '75vh', 'overflowY': 'auto'},
+            style_header={
+                'backgroundColor': '#1a1a1a',
+                'color': 'white',
+                'fontWeight': 'bold',
+                'textAlign': 'center',
+                'position': 'sticky',
+                'top': 0
+            },
+            style_data={
+                'backgroundColor': '#f8f9fa',
+                'color': '#2a2a2a',
+                'textAlign': 'center'
+            },
+            style_data_conditional=[
+                {
+                    'if': {'row_index': 'odd'},
+                    'backgroundColor': 'rgb(240, 240, 240)'
+                }
+            ],
+            style_cell={
+                'minWidth': '80px',
+                'padding': '5px',
+                'textAlign': 'center',
+                'fontFamily': 'Segoe UI, Arial, sans-serif'
+            }
+        ),
+        html.Button("Load More", id="load-more-logs-btn", n_clicks=0, style={'display': 'none'}),
+        html.Div(id='dummy-output', style={'display': 'none'})
+    ])

main.py

@@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 import polars as pl
 import dash
-from dash import dcc, html, Input, Output
+from dash import dcc, html, Input, Output, State
 import dash_bootstrap_components as dbc
 import numpy as np
 
@@ -19,56 +19,82 @@ from stats.id_viewer import _format_can_id_vec, plot_bits
 from stats.frequency import calculate_frequency, plot_frequency
 from stats.correlation import calculate_correlation, plot_correlation_heatmap
 from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap
+from logs.view import get_logs_table_component, prepare_logs_data
 
-RAW_LOG = "data/logs/rawlog.txt"
+RAW_LOG_DIR = "data/logs"
-BUS1_CSV = "data/csv/bus1.csv"
+
-BUS2_CSV = "data/csv/bus2.csv"
+def parse_vehicle_from_filename(filename: str):
-BUS1_PARQUET = "data/parquet/bus1.parquet"
+    stem = Path(filename).stem
-BUS2_PARQUET = "data/parquet/bus2.parquet"
+    if '-' in stem:
-BUS1_DECODED = "data/parquet/bus1_decoded.parquet"
+        brand, model_part = stem.split('-', 1)
-BUS2_DECODED = "data/parquet/bus2_decoded.parquet"
+    else:
+        brand, model_part = stem, "Unknown"
+    model = model_part.replace('_', ' ')
+    vehicle = f"{brand} {model}".strip()
+    return vehicle, brand, model
 
 def run_pipeline():
-    os.makedirs("data/logs", exist_ok=True)
+    os.makedirs(RAW_LOG_DIR, exist_ok=True)
     os.makedirs("data/csv", exist_ok=True)
     os.makedirs("data/parquet", exist_ok=True)
-    if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
+
-        print("Parsing raw log...")
+    for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
-        parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)
+        vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
+
+        bus1_csv = f"data/csv/{vehicle}_bus1.csv"
+        bus2_csv = f"data/csv/{vehicle}_bus2.csv"
+        bus1_parquet = f"data/parquet/{vehicle}_bus1.parquet"
+        bus2_parquet = f"data/parquet/{vehicle}_bus2.parquet"
+        bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
+        bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
+
+        if not Path(bus1_decoded).exists() or not Path(bus2_decoded).exists():
+            print(f"Parsing raw log: {log_file.name}...")
+            parse_log(str(log_file), bus1_csv, bus2_csv)
 
             print("Converting to parquet...")
-        parse_csv(BUS1_CSV).sink_parquet(BUS1_PARQUET)
+            parse_csv(bus1_csv).sink_parquet(bus1_parquet)
-        parse_csv(BUS2_CSV).sink_parquet(BUS2_PARQUET)
+            parse_csv(bus2_csv).sink_parquet(bus2_parquet)
 
             print("Decoding J1939...")
-        df1 = pl.read_parquet(BUS1_PARQUET)
+            df1 = pl.read_parquet(bus1_parquet)
-        df2 = pl.read_parquet(BUS2_PARQUET)
+            df2 = pl.read_parquet(bus2_parquet)
             dec1 = decode_j1939_frames(df1)
             dec2 = decode_j1939_frames(df2)
-        dec1.write_parquet(BUS1_DECODED)
+            dec1.write_parquet(bus1_decoded)
-        dec2.write_parquet(BUS2_DECODED)
+            dec2.write_parquet(bus2_decoded)
 
 run_pipeline()
 
 print("Loading data into memory...")
-DATA = {
+DATA = {}
-    "Bus 1": load_data(BUS1_DECODED),
+VEHICLE_META = {}
-    "Bus 2": load_data(BUS2_DECODED)
+
-}
+for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
+    vehicle, brand, model = parse_vehicle_from_filename(log_file.name)
+    VEHICLE_META[vehicle] = {"brand": brand, "model": model}
+
+    bus1_decoded = f"data/parquet/{vehicle}_bus1_decoded.parquet"
+    bus2_decoded = f"data/parquet/{vehicle}_bus2_decoded.parquet"
+
+    if Path(bus1_decoded).exists() and Path(bus2_decoded).exists():
+        DATA[vehicle] = {
+            "Bus 1": load_data(bus1_decoded),
+            "Bus 2": load_data(bus2_decoded)
+        }
 
 PRECOMPUTED_FIGURES = {}
 DATA_BY_ID = {}
 CORR_CACHE = {}
 
-def process_bus_data(bus, df):
+def process_bus_data(vehicle, bus, df):
     precomp = {}
-    precomp[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
+    precomp[f"{vehicle}_{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{vehicle} {bus} Frequency")
-    precomp[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")
+    precomp[f"{vehicle}_{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{vehicle} {bus} Byte-Level Entropy")
 
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
     df = df.assign(Formatted_ID=formatted)
-
     df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')
 
     grouped = {}
@@ -82,15 +108,17 @@ def process_bus_data(bus, df):
                 group = group.iloc[keep]
         grouped[can_id] = (group, byte_cols)
 
-    return precomp, grouped
+    return vehicle, bus, precomp, grouped
 
 with ThreadPoolExecutor() as executor:
-    futures = {executor.submit(process_bus_data, bus, df): bus for bus, df in DATA.items()}
+    futures = []
+    for vehicle, buses in DATA.items():
+        for bus, df in buses.items():
+            futures.append(executor.submit(process_bus_data, vehicle, bus, df))
     for future in futures:
-        bus = futures[future]
+        v, b, precomp, grouped = future.result()
-        precomp, grouped = future.result()
         PRECOMPUTED_FIGURES.update(precomp)
-        DATA_BY_ID[bus] = grouped
+        DATA_BY_ID[(v, b)] = grouped
 
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
 app.config.suppress_callback_exceptions = True
@@ -101,16 +129,42 @@ app.layout = dbc.Container([
         dbc.Tab(label="Overview", tab_id="overview", children=[
             html.Div(id="overview-content")
         ]),
-        dbc.Tab(label="Statistics", tab_id="statistics", children=[
+        dbc.Tab(label="Logs", tab_id="logs", children=[
             dbc.Row([
-                dbc.Col(html.Label("Select Bus:"), width=1, className="mt-2"),
+                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
                 dbc.Col(dcc.Dropdown(
-                    id='bus-selector',
+                    id='logs-vehicle-selector',
-                    options=[{'label': k, 'value': k} for k in DATA.keys()],
+                    options=[{'label': v, 'value': v} for v in DATA.keys()],
+                    value=list(DATA.keys())[0] if DATA else None,
+                    clearable=False
+                ), width=3, className="me-4"),
+                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='logs-bus-selector',
+                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
                     value='Bus 1',
                     clearable=False
                 ), width=2),
-            ], className="mb-3 mt-3"),
+            ], className="mb-3 mt-3", align="end"),
+            get_logs_table_component()
+        ]),
+        dbc.Tab(label="Statistics", tab_id="statistics", children=[
+            dbc.Row([
+                dbc.Col(html.Label("Select Vehicle:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='vehicle-selector',
+                    options=[{'label': v, 'value': v} for v in DATA.keys()],
+                    value=list(DATA.keys())[0] if DATA else None,
+                    clearable=False
+                ), width=3, className="me-4"),
+                dbc.Col(html.Label("Select Bus:", className="mt-2"), width="auto"),
+                dbc.Col(dcc.Dropdown(
+                    id='bus-selector',
+                    options=[{'label': 'Bus 1', 'value': 'Bus 1'}, {'label': 'Bus 2', 'value': 'Bus 2'}],
+                    value='Bus 1',
+                    clearable=False
+                ), width=2),
+            ], className="mb-3 mt-3", align="end"),
             dbc.Tabs([
                 dbc.Tab(label="Frequency", tab_id="freq"),
                 dbc.Tab(label="ID Viewer", tab_id="id_viewer"),
@@ -122,19 +176,103 @@ app.layout = dbc.Container([
     ], id="main-tabs", active_tab="statistics")
 ], fluid=True)
 
+app.clientside_callback(
+    """
+    function(data) {
+        if (!data) return '';
+        setTimeout(function() {
+            const btn = document.getElementById('load-more-logs-btn');
+            const info = document.getElementById('logs-info-text');
+            if (!btn || !info || info.innerText.toLowerCase().includes('all')) {
+                return;
+            }
+
+            btn.dataset.loading = "false";
+
+            const containers = document.querySelectorAll('.dash-spreadsheet-container, .dash-spreadsheet-inner');
+            if (containers.length === 0) return;
+
+            containers.forEach(container => {
+                container.onscroll = function() {
+                    if (container.scrollHeight - container.scrollTop - container.clientHeight < 200) {
+                        if (btn && btn.dataset.loading === "false") {
+                            btn.dataset.loading = "true";
+                            btn.click();
+                        }
+                    }
+                };
+
+                if (container.scrollHeight - container.scrollTop - container.clientHeight < 200) {
+                    if (btn && btn.dataset.loading === "false") {
+                        btn.dataset.loading = "true";
+                        btn.click();
+                    }
+                }
+            });
+        }, 200);
+        return '';
+    }
+    """,
+    Output('dummy-output', 'children'),
+    Input('logs-table', 'data')
+)
+
+@app.callback(
+    Output('logs-table', 'data'),
+    Output('logs-table', 'columns'),
+    Output('logs-info-text', 'children'),
+    Input('logs-vehicle-selector', 'value'),
+    Input('logs-bus-selector', 'value'),
+    Input('load-more-logs-btn', 'n_clicks'),
+    State('logs-table', 'data'),
+)
+def update_logs_table(vehicle, bus, n_clicks, current_data):
+    ctx = dash.callback_context
+    trigger_id = ctx.triggered[0]['prop_id'].split('.')[0] if ctx.triggered else ''
+
+    if not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]:
+        return [], [], "No data available"
+
+    df = DATA[vehicle][bus]
+    prepared_df = prepare_logs_data(df)
+
+    total_rows = len(prepared_df)
+    columns = [{"name": i, "id": i} for i in prepared_df.columns]
+
+    if trigger_id in ['logs-vehicle-selector', 'logs-bus-selector', '']:
+        current_data = []
+
+    offset = len(current_data) if current_data else 0
+    chunk_size = 1000
+
+    if offset >= total_rows:
+        return current_data, columns, f"Displaying all {total_rows} total frames."
+
+    next_chunk = prepared_df.iloc[offset:offset + chunk_size].to_dict('records')
+    new_data = current_data + next_chunk
+    new_offset = len(new_data)
+
+    info_text = f"Displaying {new_offset} of {total_rows} total frames."
+
+    return new_data, columns, info_text
+
 @app.callback(
     Output('tab-content', 'children'),
     Input('tabs', 'active_tab'),
+    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value')
 )
-def render_content(tab, bus):
+def render_content(tab, vehicle, bus):
-    df = DATA[bus]
+    if not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]:
+        return html.Div("No data available")
+
+    df = DATA[vehicle][bus]
 
     if tab == 'freq':
-        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_freq"], style={'height': '80vh'})
+        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_freq"], style={'height': '80vh'})
 
     elif tab == 'id_viewer':
-        ids = sorted(DATA_BY_ID[bus].keys())
+        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
         return html.Div([
             html.Label("Select CAN ID:"),
             dcc.Dropdown(
@@ -148,7 +286,7 @@ def render_content(tab, bus):
         ])
 
     elif tab == 'corr':
-        ids = sorted(DATA_BY_ID[bus].keys())
+        ids = sorted(DATA_BY_ID.get((vehicle, bus), {}).keys())
         return html.Div([
             dbc.Row([
                 dbc.Col(html.Label("Method:"), width=1, className="mt-2"),
@@ -170,53 +308,55 @@ def render_content(tab, bus):
         ])
 
     elif tab == 'entropy':
-        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{bus}_entropy"], style={'height': '80vh'})
+        return dcc.Graph(figure=PRECOMPUTED_FIGURES[f"{vehicle}_{bus}_entropy"], style={'height': '80vh'})
 
     return html.Div("Tab not found")
 
 @app.callback(
     Output('id-viewer-graph', 'figure'),
     Input('id-selector', 'value'),
+    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value'),
     Input('tabs', 'active_tab'),
 )
-def update_id_viewer(selected_id, bus, tab):
+def update_id_viewer(selected_id, vehicle, bus, tab):
-    if tab != 'id_viewer' or not selected_id:
+    if tab != 'id_viewer' or not selected_id or not vehicle or not bus:
         return dash.no_update
 
-    grouped_data = DATA_BY_ID.get(bus, {})
+    grouped_data = DATA_BY_ID.get((vehicle, bus), {})
     if selected_id not in grouped_data:
         return dash.no_update
 
     filtered_df, byte_cols = grouped_data[selected_id]
-    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{bus} Byte Visualization")
+    return plot_bits(filtered_df, byte_cols, selected_id, title=f"{vehicle} {bus} Byte Visualization")
 
 @app.callback(
     Output('corr-graph', 'figure'),
     Input('corr-method', 'value'),
     Input('corr-target', 'value'),
+    Input('vehicle-selector', 'value'),
     Input('bus-selector', 'value'),
     Input('tabs', 'active_tab'),
 )
-def update_corr(method, target, bus, tab):
+def update_corr(method, target, vehicle, bus, tab):
-    if tab != 'corr':
+    if tab != 'corr' or not vehicle or not bus:
         return dash.no_update
 
     target_id = None if target == 'all' or not target else target
-    cache_key = (bus, method, target_id)
+    cache_key = (vehicle, bus, method, target_id)
 
     if cache_key not in CORR_CACHE:
-        df = DATA[bus]
+        df = DATA[vehicle][bus]
         corr_df = calculate_correlation(df, method=method, target_id=target_id)
         CORR_CACHE[cache_key] = corr_df
     else:
         corr_df = CORR_CACHE[cache_key]
 
-    title = f"{bus} Correlation"
+    title = f"{vehicle} {bus} Correlation"
     if target_id:
         title += f" ({target_id})"
 
     return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
 
 if __name__ == '__main__':
-    app.run(debug=True)
+    app.run(debug=False)

stats/correlation.py

@@ -83,8 +83,12 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
             return c.max(axis=0)
         return np.zeros(n_cols, dtype=np.float64)
 
+    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    if len(groups) > 0:
         with ThreadPoolExecutor() as executor:
-        out = np.array(list(executor.map(_process_group, groups)))
+            results = list(executor.map(_process_group, groups))
+            for i, res in enumerate(results):
+                out[i] = res
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/entropy.py

@@ -70,8 +70,12 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
             res[ci] = _entropy_col(sub[:, ci])
         return res
 
+    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    if len(groups) > 0:
         with ThreadPoolExecutor() as executor:
-        out = np.array(list(executor.map(_process_group, groups)))
+            results = list(executor.map(_process_group, groups))
+            for i, res in enumerate(results):
+                out[i] = res
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'