From 54fd8ce74ce48281ccaf937be20a06ef88bdf3a3 Mon Sep 17 00:00:00 2001
From: Erick Ahmed <engineering@erickahmed.com>
Date: Wed, 22 Jul 2026 21:39:47 +0200
Subject: [PATCH] Implement infinite scroll for logs table

---
 logs/view.py |  6 ++--
 main.py      | 97 ++++++++++++++++++++++++++++++++++++++++------------
 2 files changed, 79 insertions(+), 24 deletions(-)

diff --git a/logs/view.py b/logs/view.py
index 8b4f83e..7bbf4d3 100644
--- a/logs/view.py
+++ b/logs/view.py
@@ -1,4 +1,4 @@
-# File: logs_view.py
+# File: logs/view.py
 # Copyright (C) 2026 Erick Ahmed
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
@@ -70,5 +70,7 @@ def get_logs_table_component():
                 'textAlign': 'center',
                 'fontFamily': 'Segoe UI, Arial, sans-serif'
             }
-        )
+        ),
+        html.Button("Load More", id="load-more-logs-btn", n_clicks=0, style={'display': 'none'}),
+        html.Div(id='dummy-output', style={'display': 'none'})
     ])
diff --git a/main.py b/main.py
index fcfb9ea..087967b 100644
--- a/main.py
+++ b/main.py
@@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 import polars as pl
 import dash
-from dash import dcc, html, Input, Output
+from dash import dcc, html, Input, Output, State
 import dash_bootstrap_components as dbc
 import numpy as np
 
@@ -19,7 +19,7 @@ from stats.id_viewer import _format_can_id_vec, plot_bits
 from stats.frequency import calculate_frequency, plot_frequency
 from stats.correlation import calculate_correlation, plot_correlation_heatmap
 from stats.entropy import calculate_byte_entropy, plot_entropy_heatmap
-from logs.view import prepare_logs_data, get_logs_table_component
+from logs.view import get_logs_table_component, prepare_logs_data
 
 RAW_LOG_DIR = "data/logs"
 
@@ -86,7 +86,6 @@ for log_file in Path(RAW_LOG_DIR).glob("*.txt"):
 PRECOMPUTED_FIGURES = {}
 DATA_BY_ID = {}
 CORR_CACHE = {}
-LOGS_CACHE = {}
 
 def process_bus_data(vehicle, bus, df):
     precomp = {}
@@ -177,31 +176,85 @@ app.layout = dbc.Container([
     ], id="main-tabs", active_tab="statistics")
 ], fluid=True)
 
-@app.callback(
-    [Output('logs-table', 'data'),
-     Output('logs-table', 'columns'),
-     Output('logs-info-text', 'children')],
-    Input('logs-vehicle-selector', 'value'),
-    Input('logs-bus-selector', 'value')
+app.clientside_callback(
+    """
+    function(data) {
+        if (!data) return '';
+        setTimeout(function() {
+            const btn = document.getElementById('load-more-logs-btn');
+            const info = document.getElementById('logs-info-text');
+            if (!btn || !info || info.innerText.toLowerCase().includes('all')) {
+                return;
+            }
+
+            btn.dataset.loading = "false";
+
+            const containers = document.querySelectorAll('.dash-spreadsheet-container, .dash-spreadsheet-inner');
+            if (containers.length === 0) return;
+
+            containers.forEach(container => {
+                container.onscroll = function() {
+                    if (container.scrollHeight - container.scrollTop - container.clientHeight < 200) {
+                        if (btn && btn.dataset.loading === "false") {
+                            btn.dataset.loading = "true";
+                            btn.click();
+                        }
+                    }
+                };
+
+                if (container.scrollHeight - container.scrollTop - container.clientHeight < 200) {
+                    if (btn && btn.dataset.loading === "false") {
+                        btn.dataset.loading = "true";
+                        btn.click();
+                    }
+                }
+            });
+        }, 200);
+        return '';
+    }
+    """,
+    Output('dummy-output', 'children'),
+    Input('logs-table', 'data')
 )
-def update_logs_table(vehicle, bus):
+
+@app.callback(
+    Output('logs-table', 'data'),
+    Output('logs-table', 'columns'),
+    Output('logs-info-text', 'children'),
+    Input('logs-vehicle-selector', 'value'),
+    Input('logs-bus-selector', 'value'),
+    Input('load-more-logs-btn', 'n_clicks'),
+    State('logs-table', 'data'),
+)
+def update_logs_table(vehicle, bus, n_clicks, current_data):
+    ctx = dash.callback_context
+    trigger_id = ctx.triggered[0]['prop_id'].split('.')[0] if ctx.triggered else ''
+
     if not vehicle or not bus or vehicle not in DATA or bus not in DATA[vehicle]:
         return [], [], "No data available"
 
-    cache_key = (vehicle, bus)
-    if cache_key not in LOGS_CACHE:
-        display_df = prepare_logs_data(DATA[vehicle][bus])
-        LOGS_CACHE[cache_key] = display_df
-    else:
-        display_df = LOGS_CACHE[cache_key]
+    df = DATA[vehicle][bus]
+    prepared_df = prepare_logs_data(df)
 
-    total_rows = len(display_df)
-    columns = [{"name": col, "id": col} for col in display_df.columns]
-    data = display_df.to_dict('records')
+    total_rows = len(prepared_df)
+    columns = [{"name": i, "id": i} for i in prepared_df.columns]
 
-    info_text = f"Displaying {total_rows} frames."
+    if trigger_id in ['logs-vehicle-selector', 'logs-bus-selector', '']:
+        current_data = []
 
-    return data, columns, info_text
+    offset = len(current_data) if current_data else 0
+    chunk_size = 1000
+
+    if offset >= total_rows:
+        return current_data, columns, f"Displaying all {total_rows} total frames."
+
+    next_chunk = prepared_df.iloc[offset:offset + chunk_size].to_dict('records')
+    new_data = current_data + next_chunk
+    new_offset = len(new_data)
+
+    info_text = f"Displaying {new_offset} of {total_rows} total frames."
+
+    return new_data, columns, info_text
 
 @app.callback(
     Output('tab-content', 'children'),
@@ -306,4 +359,4 @@ def update_corr(method, target, vehicle, bus, tab):
     return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
 
 if __name__ == '__main__':
-    app.run(debug=False)
+    app.run(debug=False)
\ No newline at end of file