Update app.py

app.py

@@ -8,7 +8,6 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 from datasets import load_dataset
-
 from sklearn.cluster import KMeans
 from sklearn.decomposition import PCA
 from sklearn.impute import SimpleImputer
@@ -19,7 +18,6 @@ from sklearn.preprocessing import StandardScaler
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# ========================= CONFIG =========================
 APP_TITLE = "Circuit Complexity Clustering"
 APP_SUBTITLE = (
     "Unsupervised grouping of quantum circuits by structural complexity "
@@ -34,16 +32,31 @@ REPO_CONFIG = {
 }
 
 NON_FEATURE_COLS = {
-    "sample_id", "sample_seed", "circuit_hash", "split", "circuit_qasm",
-    "qasm_raw", "qasm_transpiled", "circuit_type_resolved", "circuit_type_requested",
-    "noise_type", "noise_prob", "observable_bases", "observable_mode",
-    "backend_device", "precision_mode", "circuit_signature",
-    "entanglement", "meyer_wallach", "noise_label",
+    "sample_id",
+    "sample_seed",
+    "circuit_hash",
+    "split",
+    "circuit_qasm",
+    "qasm_raw",
+    "qasm_transpiled",
+    "circuit_type_resolved",
+    "circuit_type_requested",
+    "noise_type",
+    "noise_prob",
+    "observable_bases",
+    "observable_mode",
+    "backend_device",
+    "precision_mode",
+    "circuit_signature",
+    "entanglement",
+    "meyer_wallach",
+    "noise_label",
 }
 
 SOFT_EXCLUDE_PATTERNS = ["ideal_", "noisy_", "error_", "sign_ideal_", "sign_noisy_"]
 
 _ASSET_CACHE: Dict[str, pd.DataFrame] = {}
+_COMBINED_CACHE: Optional[pd.DataFrame] = None
 
 
 def safe_parse(value):
@@ -140,6 +153,18 @@ def load_dataset_df(dataset_key: str) -> pd.DataFrame:
     return _ASSET_CACHE[dataset_key]
 
 
+def load_combined_dataset(dataset_keys: List[str]) -> pd.DataFrame:
+    """Load and merge selected datasets."""
+    global _COMBINED_CACHE
+    cache_key = "|".join(sorted(dataset_keys))
+    if _COMBINED_CACHE is None or getattr(load_combined_dataset, "_cache_key", None) != cache_key:
+        frames = [load_dataset_df(key).assign(source_dataset=key) for key in dataset_keys]
+        combined = pd.concat(frames, ignore_index=True)
+        _COMBINED_CACHE = combined
+        load_combined_dataset._cache_key = cache_key  # type: ignore[attr-defined]
+    return _COMBINED_CACHE
+
+
 def load_guide_content() -> str:
     """Load the markdown guide if it exists."""
     try:
@@ -165,52 +190,130 @@ def get_available_feature_columns(df: pd.DataFrame) -> List[str]:
 def default_feature_selection(features: List[str]) -> List[str]:
     """Select a stable default feature subset."""
     preferred = [
-        "gate_entropy", "adj_density", "adj_degree_mean", "adj_degree_std",
-        "depth", "total_gates", "single_qubit_gates", "two_qubit_gates",
-        "cx_count", "qasm_length", "qasm_line_count", "qasm_gate_keyword_count",
+        "gate_entropy",
+        "adj_density",
+        "adj_degree_mean",
+        "adj_degree_std",
+        "depth",
+        "total_gates",
+        "single_qubit_gates",
+        "two_qubit_gates",
+        "cx_count",
+        "qasm_length",
+        "qasm_line_count",
+        "qasm_gate_keyword_count",
     ]
     selected = [feature for feature in preferred if feature in features]
     return selected[:10] if selected else features[:10]
 
 
+def build_dataset_profile(df: pd.DataFrame) -> str:
+    """Build a short dataset summary for the explorer tab."""
+    return (
+        f"### Dataset profile\n\n"
+        f"**Rows:** {len(df):,}  \n"
+        f"**Columns:** {len(df.columns):,}  \n"
+        f"**Available datasets:** {len(REPO_CONFIG)}"
+    )
+
+
+def refresh_explorer(dataset_key: str, split_name: str) -> Tuple[gr.update, pd.DataFrame, str, str, str, str]:
+    """Refresh the explorer view for the selected source dataset."""
+    df = load_dataset_df(dataset_key)
+    splits = df["split"].dropna().unique().tolist() if "split" in df.columns else ["train"]
+    if not splits:
+        splits = ["train"]
+
+    if split_name not in splits:
+        split_name = splits[0]
+
+    filtered = df[df["split"] == split_name] if "split" in df.columns else df
+    display_df = filtered.head(12).copy()
+
+    raw_qasm = display_df["qasm_raw"].iloc[0] if "qasm_raw" in display_df.columns and not display_df.empty else "// N/A"
+    transpiled_qasm = display_df["qasm_transpiled"].iloc[0] if "qasm_transpiled" in display_df.columns and not display_df.empty else "// N/A"
+
+    profile_box = build_dataset_profile(df)
+    summary_box = (
+        f"### Split summary\n\n"
+        f"**Dataset:** `{dataset_key}`  \n"
+        f"**Available splits:** {', '.join(splits)}  \n"
+        f"**Preview rows:** {len(display_df)}"
+    )
+
+    return (
+        gr.update(choices=splits, value=split_name),
+        display_df,
+        raw_qasm,
+        transpiled_qasm,
+        profile_box,
+        summary_box,
+    )
+
+
+def sync_feature_picker(dataset_keys: List[str]) -> gr.update:
+    """Refresh the feature list from the selected datasets."""
+    if not dataset_keys:
+        return gr.update(choices=[], value=[])
+
+    df = load_combined_dataset(dataset_keys)
+    features = get_available_feature_columns(df)
+    defaults = default_feature_selection(features)
+    return gr.update(choices=features, value=defaults)
+
+
 def run_clustering(
-    dataset_key: str,
+    dataset_keys: List[str],
     feature_columns: List[str],
     n_clusters: int,
     random_state: float,
 ) -> Tuple[Optional[plt.Figure], str, pd.DataFrame]:
-    """Run K-Means clustering and return PCA plot + metrics."""
+    """Run K-Means clustering and return a PCA plot plus metrics."""
+    if not dataset_keys:
+        return None, "### ❌ Please select at least one dataset.", pd.DataFrame()
+
     if not feature_columns:
-        return None, "### ❌ Please select at least one feature.", None
+        return None, "### ❌ Please select at least one feature.", pd.DataFrame()
 
-    df = load_dataset_df(dataset_key)
+    df = load_combined_dataset(dataset_keys)
     train_df = df.dropna(subset=feature_columns).copy()
 
     if len(train_df) < 30:
-        return None, "### ❌ Not enough rows after filtering missing values.", None
+        return None, "### ❌ Not enough rows after filtering missing values.", pd.DataFrame()
 
     X = train_df[feature_columns]
-
-    pipeline = Pipeline([
-        ("imputer", SimpleImputer(strategy="median")),
-        ("scaler", StandardScaler()),
-        ("pca", PCA(n_components=2, random_state=int(random_state))),
-        ("kmeans", KMeans(n_clusters=n_clusters, random_state=int(random_state), n_init=10))
-    ])
+    seed = int(random_state)
+
+    pipeline = Pipeline(
+        steps=[
+            ("imputer", SimpleImputer(strategy="median")),
+            ("scaler", StandardScaler()),
+            ("pca", PCA(n_components=2, random_state=seed)),
+            ("kmeans", KMeans(n_clusters=n_clusters, random_state=seed, n_init=10)),
+        ]
+    )
 
     pipeline.fit(X)
     labels = pipeline.named_steps["kmeans"].labels_
-    pca_coords = pipeline.named_steps["pca"].transform(
-        pipeline.named_steps["scaler"].transform(
-            pipeline.named_steps["imputer"].transform(X)
-        )
-    )
 
-    sil_score = silhouette_score(X, labels)
+    transformed = pipeline.named_steps["imputer"].transform(X)
+    transformed = pipeline.named_steps["scaler"].transform(transformed)
+    pca_coords = pipeline.named_steps["pca"].transform(transformed)
+
+    try:
+        sil_score = silhouette_score(X, labels)
+    except Exception:
+        sil_score = float("nan")
 
-    # Plot
     fig, ax = plt.subplots(figsize=(10, 8))
-    scatter = ax.scatter(pca_coords[:, 0], pca_coords[:, 1], c=labels, cmap="tab10", s=30, alpha=0.8)
+    scatter = ax.scatter(
+        pca_coords[:, 0],
+        pca_coords[:, 1],
+        c=labels,
+        cmap="tab10",
+        s=30,
+        alpha=0.8,
+    )
     ax.set_title(f"Circuit Complexity Clusters (K={n_clusters})")
     ax.set_xlabel("PCA Component 1")
     ax.set_ylabel("PCA Component 2")
@@ -218,21 +321,29 @@ def run_clustering(
     plt.colorbar(scatter, ax=ax, label="Cluster")
     plt.tight_layout()
 
-    # Cluster summary
     summary = train_df.copy()
     summary["cluster"] = labels
-    cluster_summary = summary.groupby("cluster").size().reset_index()
-    cluster_summary.columns = ["Cluster", "Number of Circuits"]
+    cluster_summary = summary.groupby("cluster").size().reset_index(name="Number of Circuits")
+
+    dataset_counts = (
+        summary.groupby(["cluster", "source_dataset"]).size().reset_index(name="Count")
+        if "source_dataset" in summary.columns
+        else pd.DataFrame()
+    )
 
     metrics_text = (
         f"### Clustering Results\n\n"
-        f"**Number of circuits clustered:** {len(train_df):,}\n"
-        f"**Number of clusters:** {n_clusters}\n"
+        f"**Datasets used:** {', '.join(dataset_keys)}  \n"
+        f"**Number of circuits clustered:** {len(train_df):,}  \n"
+        f"**Number of clusters:** {n_clusters}  \n"
         f"**Silhouette Score:** {sil_score:.4f} (closer to 1 = better separation)\n\n"
         f"**Cluster sizes:**\n"
         f"{cluster_summary.to_markdown(index=False)}"
     )
 
+    if not dataset_counts.empty:
+        metrics_text += f"\n\n**Dataset composition per cluster:**\n{dataset_counts.to_markdown(index=False)}"
+
     return fig, metrics_text, cluster_summary
 
 
@@ -270,14 +381,18 @@ with gr.Blocks(title=APP_TITLE) as demo:
                 transpiled_qasm = gr.Code(label="Transpiled QASM", language=None)
 
         with gr.TabItem("🧠 Clustering"):
+            dataset_picker = gr.CheckboxGroup(
+                label="Datasets",
+                choices=list(REPO_CONFIG.keys()),
+                value=list(REPO_CONFIG.keys()),
+            )
             feature_picker = gr.CheckboxGroup(label="Input features", choices=[])
             n_clusters = gr.Slider(2, 10, value=4, step=1, label="Number of Clusters")
             seed = gr.Number(value=42, precision=0, label="Random Seed")
             run_btn = gr.Button("🚀 Run K-Means Clustering", variant="primary")
-
             plot = gr.Plot()
             metrics = gr.Markdown()
-            cluster_table = gr.Dataframe(label="Cluster Sizes")
+            cluster_table = gr.Dataframe(label="Cluster Sizes", interactive=False)
 
         with gr.TabItem("📖 Guide"):
             gr.Markdown(load_guide_content())
@@ -290,35 +405,6 @@ with gr.Blocks(title=APP_TITLE) as demo:
         "[GitHub](https://github.com/QSBench)"
     )
 
-    # Callbacks
-    def refresh_explorer(dataset_key: str, split_name: str):
-        df = load_dataset_df(dataset_key)
-        splits = df["split"].dropna().unique().tolist() if "split" in df.columns else ["train"]
-        if not splits:
-            splits = ["train"]
-        if split_name not in splits:
-            split_name = splits[0]
-        filtered = df[df["split"] == split_name] if "split" in df.columns else df
-        display_df = filtered.head(12).copy()
-        raw = display_df["qasm_raw"].iloc[0] if not display_df.empty else "// N/A"
-        transpiled = display_df["qasm_transpiled"].iloc[0] if not display_df.empty else "// N/A"
-        profile = f"### Dataset profile\n\n**Rows:** {len(df):,}\n**Columns:** {len(df.columns):,}"
-        summary = f"### Split summary\n\n**Dataset:** `{dataset_key}`\n**Available splits:** {', '.join(splits)}\n**Preview rows:** {len(display_df)}"
-        return (
-            gr.update(choices=splits, value=split_name),
-            display_df,
-            raw,
-            transpiled,
-            profile,
-            summary,
-        )
-
-    def sync_feature_picker(dataset_key: str):
-        df = load_dataset_df(dataset_key)
-        features = get_available_feature_columns(df)
-        defaults = default_feature_selection(features)
-        return gr.update(choices=features, value=defaults)
-
     dataset_dropdown.change(
         refresh_explorer,
         [dataset_dropdown, split_dropdown],
@@ -329,11 +415,12 @@ with gr.Blocks(title=APP_TITLE) as demo:
         [dataset_dropdown, split_dropdown],
         [split_dropdown, explorer_df, raw_qasm, transpiled_qasm, profile_box, summary_box],
     )
-    dataset_dropdown.change(sync_feature_picker, [dataset_dropdown], [feature_picker])
+
+    dataset_picker.change(sync_feature_picker, [dataset_picker], [feature_picker])
 
     run_btn.click(
         run_clustering,
-        [dataset_dropdown, feature_picker, n_clusters, seed],
+        [dataset_picker, feature_picker, n_clusters, seed],
         [plot, metrics, cluster_table],
     )
 
@@ -342,8 +429,8 @@ with gr.Blocks(title=APP_TITLE) as demo:
         [dataset_dropdown, split_dropdown],
         [split_dropdown, explorer_df, raw_qasm, transpiled_qasm, profile_box, summary_box],
     )
-    demo.load(sync_feature_picker, [dataset_dropdown], [feature_picker])
+    demo.load(sync_feature_picker, [dataset_picker], [feature_picker])
 
 
 if __name__ == "__main__":
-    demo.launch(theme=gr.themes.Soft(), css=CUSTOM_CSS)
+    demo.launch(theme=gr.themes.Soft(), css=CUSTOM_CSS)