can we precompute and cache features for all CV ?

[guillaume-osmo]

We recompute 25 times the same features which suboptimal for the 5x5 CV.

Can we cached them in order to use for example with import joblib ?

import os
import joblib
import pandas as pd
from tqdm import tqdm
from calc_osmordred import CalcOsmordred  
from concurrent.futures import ProcessPoolExecutor, as_completed

# Set up caching
CACHE_DIR = "./feature_cache"
os.makedirs(CACHE_DIR, exist_ok=True)
memory = joblib.Memory(CACHE_DIR, verbose=0)

def compute_osmordred(smiles):
    return CalcOsmordred(smiles)


def load_or_compute_features(df, version=2, n_jobs=4):
    """Loads cached features or computes them if missing, preserving indices for CV splits."""
    feature_file = f"{CACHE_DIR}/osmordred_features_v{version}.csv"

    #  Check if cached features exist
    if os.path.exists(feature_file) and os.path.getsize(feature_file) > 0:
        print(" Loading cached features...")
        feature_df = pd.read_csv(feature_file)

        #  Ensure feature_df has an index column
        if "Index" not in feature_df.columns:
            raise RuntimeError("Cached feature file is missing 'Index' column!")

        feature_df.set_index("Index", inplace=True)  # Restore original indices
        return feature_df

    print("Computing new features...")
    results = []

    with ProcessPoolExecutor(max_workers=n_jobs) as executor:
        futures = {executor.submit(compute_osmordred, smi): idx for idx, smi in df.SMILES.items()}

        for future in tqdm(as_completed(futures), total=len(futures), desc="Computing Features"):
            idx = futures[future]  # Get dataset index
            try:
                result = future.result()
                if result is not None:
                    results.append((idx, result))
            except Exception as e:
                print(f"Error processing index {idx}: {e}")

    if results:
        feature_df = pd.DataFrame([res[1] for res in sorted(results, key=lambda x: x[0])])
        feature_df.insert(0, "Index", [res[0] for res in sorted(results, key=lambda x: x[0])])
        feature_df.to_csv(feature_file, index=False)
        feature_df.set_index("Index", inplace=True)  # Ensure proper indexing
        return feature_df
    else:
        raise RuntimeError("All feature computations failed! No data to save.")

def get_features_for_split(split_df, all_features_df):
    """Selects the correct precomputed features for a given CV split."""
    return all_features_df.loc[split_df.index]

[PatWalters]

Thanks, this is something I was planning to do. I'll incorporate your changes into the code.

[guillaume-osmo]

Great, this is an example code that works on my case, but more unit testing are required.