Parallelize data processing tasks with ThreadPoolExecutor

main.py

@@ -4,6 +4,7 @@
 
 import os
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import polars as pl
 import dash
@@ -59,9 +60,10 @@ PRECOMPUTED_FIGURES = {}
 DATA_BY_ID = {}
 CORR_CACHE = {}
 
-for bus, df in DATA.items():
+def process_bus_data(bus, df):
-    PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
+    precomp = {}
-    PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")
+    precomp[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency")
+    precomp[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy")
 
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
@@ -80,7 +82,15 @@ for bus, df in DATA.items():
                 group = group.iloc[keep]
         grouped[can_id] = (group, byte_cols)
 
-    DATA_BY_ID[bus] = grouped
+    return precomp, grouped
+
+with ThreadPoolExecutor() as executor:
+    futures = {executor.submit(process_bus_data, bus, df): bus for bus, df in DATA.items()}
+    for future in futures:
+        bus = futures[future]
+        precomp, grouped = future.result()
+        PRECOMPUTED_FIGURES.update(precomp)
+        DATA_BY_ID[bus] = grouped
 
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
 app.config.suppress_callback_exceptions = True
@@ -209,4 +219,4 @@ def update_corr(method, target, bus, tab):
     return plot_correlation_heatmap(corr_df, target_id=target_id, title=title)
 
 if __name__ == '__main__':
-    app.run(debug=False)
+    app.run(debug=True)

stats/correlation.py

@@ -4,6 +4,7 @@
 
 import argparse
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -69,8 +70,7 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
     else:
         groups = []
 
-    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    def _process_group(sub):
-    for gi, sub in enumerate(groups):
         mask = ~np.isnan(sub).any(axis=1)
         sub = sub[mask]
         if len(sub) > 1:
@@ -80,7 +80,11 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None =
                 c = np.abs(np.corrcoef(sub, rowvar=False))
             np.nan_to_num(c, copy=False, nan=0.0)
             np.fill_diagonal(c, 0.0)
-            out[gi] = c.max(axis=0)
+            return c.max(axis=0)
+        return np.zeros(n_cols, dtype=np.float64)
+
+    with ThreadPoolExecutor() as executor:
+        out = np.array(list(executor.map(_process_group, groups)))
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'

stats/entropy.py

@@ -4,6 +4,7 @@
 
 import argparse
 from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
 import numpy as np
 import pandas as pd
@@ -63,10 +64,14 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
     else:
         groups = []
 
-    out = np.zeros((len(unique_ids), n_cols), dtype=np.float64)
+    def _process_group(sub):
-    for gi, sub in enumerate(groups):
+        res = np.zeros(n_cols, dtype=np.float64)
         for ci in range(n_cols):
-            out[gi, ci] = _entropy_col(sub[:, ci])
+            res[ci] = _entropy_col(sub[:, ci])
+        return res
+
+    with ThreadPoolExecutor() as executor:
+        out = np.array(list(executor.map(_process_group, groups)))
 
     result = pd.DataFrame(out, index=unique_ids, columns=available_cols)
     result.index.name = 'Identifier'