Explicitly specify grouping column in dataframe iteration

main.py

@@ -29,15 +29,15 @@ BUS2_DECODED = "data/parquet/bus2_decoded.parquet"
 
 def run_pipeline():
     os.makedirs("data/logs", exist_ok=True)
+    os.makedirs("data/csv", exist_ok=True)
+    os.makedirs("data/parquet", exist_ok=True)
     if not Path(BUS1_DECODED).exists() or not Path(BUS2_DECODED).exists():
         print("Parsing raw log...")
         parse_log(RAW_LOG, BUS1_CSV, BUS2_CSV)
 
         print("Converting to parquet...")
-        lf1 = parse_csv(BUS1_CSV)
-        lf1.sink_parquet(BUS1_PARQUET)
-        lf2 = parse_csv(BUS2_CSV)
-        lf2.sink_parquet(BUS2_PARQUET)
+        parse_csv(BUS1_CSV).sink_parquet(BUS1_PARQUET)
+        parse_csv(BUS2_CSV).sink_parquet(BUS2_PARQUET)
 
         print("Decoding J1939...")
         df1 = pl.read_parquet(BUS1_PARQUET)
@@ -70,7 +70,7 @@ for bus, df in DATA.items():
     df = df.sort_values(['Formatted_ID', 'Timestamp'], kind='stable')
 
     grouped = {}
-    for can_id, group in df.groupby('Formatted_ID'):
+    for can_id, group in df.groupby(by='Formatted_ID'):
         byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in group.columns]
         if not group.empty and len(byte_cols) > 0:
             arr = group[byte_cols].to_numpy(dtype=np.float32, copy=False)

stats/correlation.py

@@ -27,8 +27,7 @@ def _ensure_int_bytes(df: pd.DataFrame, cols: list) -> pd.DataFrame:
     return df
 
 def calculate_correlation(df: pd.DataFrame, method: str, target_id: str | None = None) -> pd.DataFrame:
-    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
-    available_cols = [col for col in byte_cols if col in df.columns]
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
 
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")

stats/entropy.py

@@ -36,8 +36,7 @@ def _entropy_col(a: np.ndarray) -> float:
     return float(-np.sum(p * np.log2(p)))
 
 def calculate_byte_entropy(df: pd.DataFrame) -> pd.DataFrame:
-    byte_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
-    available_cols = byte_cols
+    available_cols = [f"b{i}" for i in range(8) if f"b{i}" in df.columns]
     if not available_cols:
         raise ValueError("No byte columns (b0-b7) found in the DataFrame")
 

stats/frequency.py

@@ -18,7 +18,6 @@ def _format_can_id_vec(s: pd.Series) -> pd.Series:
 def calculate_frequency(df: pd.DataFrame) -> pd.DataFrame:
     can_id_col = 'ID' if 'ID' in df.columns else 'Identifier'
     formatted = _format_can_id_vec(df[can_id_col])
-    df['Formatted_ID'] = formatted
 
     counts = formatted.value_counts()
     freq_df = pd.DataFrame({