diff --git a/main.py b/main.py index 70bf667..68cd788 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ import os from pathlib import Path +from concurrent.futures import ThreadPoolExecutor import polars as pl import dash @@ -59,9 +60,10 @@ PRECOMPUTED_FIGURES = {} DATA_BY_ID = {} CORR_CACHE = {} -for bus, df in DATA.items(): - PRECOMPUTED_FIGURES[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency") - PRECOMPUTED_FIGURES[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy") +def process_bus_data(bus, df): + precomp = {} + precomp[f"{bus}_freq"] = plot_frequency(calculate_frequency(df), title=f"{bus} Frequency") + precomp[f"{bus}_entropy"] = plot_entropy_heatmap(calculate_byte_entropy(df), title=f"{bus} Byte-Level Entropy") can_id_col = 'ID' if 'ID' in df.columns else 'Identifier' formatted = _format_can_id_vec(df[can_id_col]) @@ -80,7 +82,15 @@ for bus, df in DATA.items(): group = group.iloc[keep] grouped[can_id] = (group, byte_cols) - DATA_BY_ID[bus] = grouped + return precomp, grouped + +with ThreadPoolExecutor() as executor: + futures = {executor.submit(process_bus_data, bus, df): bus for bus, df in DATA.items()} + for future in futures: + bus = futures[future] + precomp, grouped = future.result() + PRECOMPUTED_FIGURES.update(precomp) + DATA_BY_ID[bus] = grouped app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) app.config.suppress_callback_exceptions = True @@ -209,4 +219,4 @@ def update_corr(method, target, bus, tab): return plot_correlation_heatmap(corr_df, target_id=target_id, title=title) if __name__ == '__main__': - app.run(debug=False) + app.run(debug=True) diff --git a/stats/correlation.py b/stats/correlation.py index f6003d6..3035f39 100644 --- a/stats/correlation.py +++ b/stats/correlation.py @@ -4,6 +4,7 @@ import argparse from pathlib import Path +from concurrent.futures import ThreadPoolExecutor import numpy as np import pandas as pd @@ -69,8 +70,7 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = else: groups = [] - out = np.zeros((len(unique_ids), n_cols), dtype=np.float64) - for gi, sub in enumerate(groups): + def _process_group(sub): mask = ~np.isnan(sub).any(axis=1) sub = sub[mask] if len(sub) > 1: @@ -80,7 +80,11 @@ def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = c = np.abs(np.corrcoef(sub, rowvar=False)) np.nan_to_num(c, copy=False, nan=0.0) np.fill_diagonal(c, 0.0) - out[gi] = c.max(axis=0) + return c.max(axis=0) + return np.zeros(n_cols, dtype=np.float64) + + with ThreadPoolExecutor() as executor: + out = np.array(list(executor.map(_process_group, groups))) result = pd.DataFrame(out, index=unique_ids, columns=available_cols) result.index.name = 'Identifier' diff --git a/stats/entropy.py b/stats/entropy.py index df42d15..ac3295e 100644 --- a/stats/entropy.py +++ b/stats/entropy.py @@ -4,6 +4,7 @@ import argparse from pathlib import Path +from concurrent.futures import ThreadPoolExecutor import numpy as np import pandas as pd @@ -63,10 +64,14 @@ def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame: else: groups = [] - out = np.zeros((len(unique_ids), n_cols), dtype=np.float64) - for gi, sub in enumerate(groups): + def _process_group(sub): + res = np.zeros(n_cols, dtype=np.float64) for ci in range(n_cols): - out[gi, ci] = _entropy_col(sub[:, ci]) + res[ci] = _entropy_col(sub[:, ci]) + return res + + with ThreadPoolExecutor() as executor: + out = np.array(list(executor.map(_process_group, groups))) result = pd.DataFrame(out, index=unique_ids, columns=available_cols) result.index.name = 'Identifier'