diff --git a/app.py b/app.py
index 97f9a16..e76dcd8 100644
--- a/app.py
+++ b/app.py
@@ -1,3 +1,10 @@
+import os
+# Polars' default (CPU-core-count-sized) native thread pool access-violation
+# crashes the whole process on some high-core-count Windows machines when
+# invoked from Streamlit's script-runner thread. Must be set before polars
+# is imported anywhere (including transitively via openms_insight).
+os.environ.setdefault("POLARS_MAX_THREADS", "1")
+
import streamlit as st
from pathlib import Path
import json
@@ -23,12 +30,19 @@
st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="๐"),
st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="๐ฏ"),
st.Page(Path("content", "results_abundance.py"), title="Abundance", icon="๐"),
+
+ ],
+ "Differential Protein Analysis": [
+ st.Page(Path("content", "filtering.py"), title="Filtering", icon="๐งน"),
+ st.Page(Path("content", "imputation.py"), title="Imputation", icon="๐ฉน"),
+ st.Page(Path("content", "normalization.py"), title="Normalization", icon="โ๏ธ"),
+ st.Page(Path("content", "statistical.py"), title="Statistical", icon="๐ข"),
st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="๐"),
st.Page(Path("content", "results_pca.py"), title="PCA", icon="๐"),
st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="๐ฅ"),
- st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="๐"),
- st.Page(Path("content", "results_pathway_analysis.py"), title="Pathway Analysis", icon="๐"),
- ],
+ st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered Heatmap", icon="๐งฌ"),
+ st.Page(Path("content", "enrichment.py"), title="Pathway Analysis", icon="๐"),
+ ]
}
pg = st.navigation(pages)
diff --git a/content/enrichment.py b/content/enrichment.py
new file mode 100644
index 0000000..62fc9fa
--- /dev/null
+++ b/content/enrichment.py
@@ -0,0 +1,141 @@
+"""Pathway Analysis Page."""
+
+from pathlib import Path
+import pandas as pd
+import polars as pl
+import streamlit as st
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column
+# Import GO Enrichment modules from openms_insight engine
+from openms_insight.analysis.enrichment import calculate_go_enrichment
+
+params = page_setup()
+st.title("GO Enrichment Analysis")
+
+st.markdown(
+ """
+Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+# --- STEP 1: Upstream Statistics Checkpoint ---
+if (
+ "statistics_df" in st.session_state
+ and st.session_state["statistics_df"] is not None
+):
+ final_statistics_report = st.session_state["statistics_df"]
+ st.info(
+ "๐ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step."
+ )
+else:
+ st.warning(
+ "โ ๏ธ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first."
+ )
+ st.page_link(
+ "content/statistical.py", label="Go to Statistical Inference", icon="๐ฌ"
+ )
+ st.stop()
+
+# --- STEP 2: Preprocessing Mapping Key Configuration ---
+# Identify target identifier columns dynamically
+abundance_result = get_abundance_data(st.session_state["workspace"])
+id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName"
+if id_col not in final_statistics_report.columns:
+ st.error(f"โ Structural Error: Column '{id_col}' is missing from the active matrix context.")
+ st.stop()
+
+# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels ---
+st.subheader("Configure Enrichment Thresholds")
+
+# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj')
+target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value"
+p_label = (
+ "Adjusted P-value (p-adj) Cutoff"
+ if target_p_col == "p-adj"
+ else "Raw P-value (p-value) Cutoff"
+)
+
+ui_go_col1, ui_go_col2 = st.columns(2)
+
+with ui_go_col1:
+ p_cutoff = st.number_input(
+ f"๐ฌ {p_label}",
+ min_value=0.0001,
+ max_value=1.0,
+ value=0.05,
+ step=0.01,
+ format="%.4f",
+ help="Proteins with significance metrics below this value are mapped to the foreground cohort.",
+ )
+
+with ui_go_col2:
+ fc_cutoff = st.number_input(
+ "๐ Absolute Difference Cutoff (|log2FC|)",
+ min_value=0.0,
+ max_value=10.0,
+ value=1.0,
+ step=0.1,
+ format="%.2f",
+ help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.",
+ )
+
+# --- SECTION 2: Execution and Interactive View Charts ---
+st.markdown("
", unsafe_allow_html=True)
+if st.button("๐ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"):
+
+ with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."):
+ # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation
+ stats_pl = pl.from_pandas(final_statistics_report)
+
+ status, output = calculate_go_enrichment(
+ final_report=stats_pl,
+ id_col=id_col,
+ target_p_col=target_p_col,
+ p_cutoff=p_cutoff,
+ fc_cutoff=fc_cutoff,
+ )
+
+ # Route response structures based on analysis output status code
+ if status == "empty_data":
+ st.error("โ No valid statistical rows found containing standard columns to run GO alignment.")
+
+ elif status == "insufficient_proteins":
+ st.warning(
+ f"โ ๏ธ Not enough significant proteins found to construct target datasets. "
+ f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| โฅ {fc_cutoff:.2f})."
+ )
+ st.info(f"๐ก Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.")
+
+ elif status == "success":
+ st.success("โญ GO Enrichment Analysis completed successfully!")
+
+ # Display operational matrix scale
+ st.markdown(
+ f"๐ **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items."
+ )
+
+ # Build multi-tab interface layer for ontology subcategories
+ tabs = st.tabs([
+ "๐งฌ Biological Process (BP)",
+ "๐ฌ Cellular Component (CC)",
+ "๐งช Molecular Function (MF)"
+ ])
+ categories_data = output["categories"]
+
+ for idx, go_type in enumerate(["BP", "CC", "MF"]):
+ with tabs[idx]:
+ fig = categories_data[go_type]["fig"]
+ df_go = categories_data[go_type]["df"]
+
+ if fig is not None and df_go is not None:
+ # Render plotly bar figures generated straight from backend engine
+ st.plotly_chart(fig, use_container_width=True)
+
+ st.subheader(f"๐ {go_type} Results Dataframe")
+ st.dataframe(df_go, use_container_width=True)
+ else:
+ st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**")
\ No newline at end of file
diff --git a/content/filtering.py b/content/filtering.py
new file mode 100644
index 0000000..2bad00c
--- /dev/null
+++ b/content/filtering.py
@@ -0,0 +1,173 @@
+"""Filtering Page."""
+
+from pathlib import Path
+import pandas as pd
+import polars as pl
+import streamlit as st
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+
+# Import filtering functions from openms_insight package
+from openms_insight.analysis.filter import (
+ filter_low_abundance,
+ filter_low_repeatability,
+ filter_low_variance,
+)
+
+STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"]
+
+
+def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
+ """Keep preprocessing tables intensity-only before statistical analysis."""
+ return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore")
+
+params = page_setup()
+st.title("Data Filtering")
+
+st.markdown(
+ """
+Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+ st.info(
+ "Abundance data not available. Please run the workflow and configure sample groups first."
+ )
+ st.page_link(
+ "content/results_abundance.py", label="Go to Abundance", icon="๐"
+ )
+ st.stop()
+
+pivot_df, expr_df, group_map = result
+pivot_df = strip_stat_columns(pivot_df)
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+# 1. Identify actual sample columns dynamically
+sample_cols = [
+ c
+ for c in pivot_df.columns
+ if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"]
+]
+
+# --- SECTION 1: Original Data View ---
+st.subheader("Original Abundance Table")
+st.markdown(
+ f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering."
+)
+st.dataframe(pivot_df, use_container_width=True)
+
+st.markdown("---")
+
+# --- SECTION 2: Filter Configuration ---
+st.subheader("Configure Filter Engine")
+
+# Prepare Polars Metadata DataFrame required by openms_insight functions
+metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map]
+metadata_pl = pl.DataFrame(
+ metadata_rows, schema={"sample_id": pl.String, "group": pl.String}
+)
+
+# User selection for filtering strategy
+filter_method = st.selectbox(
+ "Select Filtering Method",
+ options=["Low Abundance", "Low Repeatability", "Low Variance"],
+ index=0,
+ help="Choose the statistical criteria to prune unreliable protein entries.",
+)
+
+# Render threshold sliders dynamically based on the selected filter method
+if filter_method == "Low Abundance":
+ st.markdown(
+ "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold."
+ )
+ threshold = st.slider(
+ "Threshold Percentile (%)",
+ min_value=0.0,
+ max_value=100.0,
+ value=10.0,
+ step=5.0,
+ )
+
+elif filter_method == "Low Repeatability":
+ st.markdown(
+ "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum."
+ )
+ threshold = st.slider(
+ "Max Missing Ratio",
+ min_value=0.0,
+ max_value=100.0,
+ value=50.0,
+ step=5.0,
+ help="Allowed missing value (zero or null) ratio per group.",
+ )
+
+elif filter_method == "Low Variance":
+ st.markdown(
+ "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold."
+ )
+ threshold = st.slider(
+ "Threshold Percentile (%)",
+ min_value=0.0,
+ max_value=100.0,
+ value=10.0,
+ step=5.0,
+ )
+
+# --- SECTION 3: Filter Execution and Collected Results View ---
+if st.button("Apply Filter", type="primary"):
+ # Convert the original Pandas DataFrame into a Polars LazyFrame graph
+ quant_lazy = pl.from_pandas(pivot_df).lazy()
+
+ # Route execution to the chosen openms_insight engine function
+ if filter_method == "Low Abundance":
+ filtered_lazy = filter_low_abundance(
+ quantification_data=quant_lazy,
+ metadata=metadata_pl,
+ group_column="group",
+ threshold_percentile=threshold,
+ )
+ elif filter_method == "Low Repeatability":
+ # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5)
+ filtered_lazy = filter_low_repeatability(
+ quantification_data=quant_lazy,
+ metadata=metadata_pl,
+ group_column="group",
+ max_missing_ratio=threshold / 100.0,
+ )
+ elif filter_method == "Low Variance":
+ filtered_lazy = filter_low_variance(
+ quantification_data=quant_lazy,
+ metadata=metadata_pl,
+ group_column="group",
+ threshold_percentile=threshold,
+ )
+
+ # Collect the evaluated lazy graph and convert back to Pandas for visualization
+ filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas())
+ st.session_state["filtered_df"] = filtered_df
+
+ # Layout response metrics and the filtered matrix
+ st.success(f"Successfully applied **{filter_method}** filter!")
+
+ # Display dataset scale compression stats
+ col1, col2, col3 = st.columns(3)
+ col1.metric("Original Proteins", pivot_df.shape[0])
+ col2.metric("Filtered Proteins", filtered_df.shape[0])
+ col3.metric(
+ "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None
+ )
+
+ st.subheader("Filtered Abundance Table")
+ if filtered_df.empty:
+ st.warning(
+ "The filtered table is empty. Try relaxing the threshold constraints."
+ )
+ else:
+ st.dataframe(filtered_df, use_container_width=True)
\ No newline at end of file
diff --git a/content/imputation.py b/content/imputation.py
new file mode 100644
index 0000000..4350263
--- /dev/null
+++ b/content/imputation.py
@@ -0,0 +1,145 @@
+"""Imputation Page."""
+
+from pathlib import Path
+import pandas as pd
+import polars as pl
+import streamlit as st
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+
+# Import imputation algorithms from openms_insight engine
+from openms_insight.analysis.imputation import impute_mar, impute_smallest_value
+
+STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"]
+
+
+def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame:
+ """Keep preprocessing tables intensity-only before statistical analysis."""
+ return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore")
+
+params = page_setup()
+st.title("Missing Value Imputation")
+
+st.markdown(
+ """
+Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+# Load base dataset and clean dictionary keys
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+ st.info(
+ "Abundance data not available. Please run the workflow and configure sample groups first."
+ )
+ st.page_link(
+ "content/results_abundance.py", label="Go to Abundance", icon="๐"
+ )
+ st.stop()
+
+pivot_df, expr_df, group_map = result
+pivot_df = strip_stat_columns(pivot_df)
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix
+if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None:
+ base_df = strip_stat_columns(st.session_state["filtered_df"])
+ st.session_state["filtered_df"] = base_df
+ st.info(
+ "๐ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step."
+ )
+else:
+ base_df = pivot_df
+ st.warning(
+ "โ ๏ธ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table."
+ )
+
+# 2. Identify actual sample columns dynamically based on the current active matrix
+sample_cols = [
+ c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"]
+]
+
+# --- SECTION 1: Input Matrix Summary ---
+st.subheader("Input Matrix Overview")
+st.markdown(
+ f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation."
+)
+st.dataframe(base_df, use_container_width=True)
+
+st.markdown("---")
+
+# --- SECTION 2: Imputation Configuration ---
+st.subheader("Configure Imputation Engine")
+
+# Build Polars structural metadata DataFrame
+metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map]
+metadata_pl = pl.DataFrame(
+ metadata_rows, schema={"sample_id": pl.String, "group": pl.String}
+)
+
+# User selection for core missingness assumption strategy
+impute_category = st.selectbox(
+ "Select Imputation Class",
+ options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"],
+ index=0,
+ help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.",
+)
+
+# Render algorithmic options sub-menus based on the parent selection
+if impute_category == "MAR (Missing At Random)":
+ st.markdown(
+ "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group."
+ )
+ strategy_opt = st.radio(
+ "Mathematical Strategy",
+ options=["median", "mean"],
+ index=0,
+ horizontal=True,
+ )
+
+elif impute_category == "MNAR (Missing Not At Random)":
+ st.markdown(
+ "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits."
+ )
+ scope_opt = st.radio(
+ "Detection Minimum Scope",
+ options=["row", "global"],
+ index=0,
+ horizontal=True,
+ help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.",
+ )
+
+# --- SECTION 3: Imputation Execution ---
+if st.button("Apply Imputation", type="primary"):
+ # Initialize optimization pipeline graph via lazy loading conversion
+ quant_lazy = pl.from_pandas(base_df).lazy()
+
+ # Route configuration matrix parameters to designated engine function channels
+ if impute_category == "MAR (Missing At Random)":
+ imputed_lazy = impute_mar(
+ quantification_data=quant_lazy,
+ metadata=metadata_pl,
+ group_column="group",
+ strategy=strategy_opt,
+ )
+ elif impute_category == "MNAR (Missing Not At Random)":
+ imputed_lazy = impute_smallest_value(
+ quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt
+ )
+
+ # Resolve lazy graph optimization tree and push to display data frame structure
+ imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas())
+
+ # ๐พ Save current output into Session State for down-stream processing (Normalization, Statistics)
+ st.session_state["imputed_df"] = imputed_df
+
+ st.success(f"Successfully finalized **{impute_category}** imputation step!")
+
+ # Calculate and display a quick performance matrix check
+ st.subheader("Imputed Result Table")
+ st.dataframe(imputed_df, use_container_width=True)
\ No newline at end of file
diff --git a/content/normalization.py b/content/normalization.py
new file mode 100644
index 0000000..c0e97e8
--- /dev/null
+++ b/content/normalization.py
@@ -0,0 +1,242 @@
+"""Normalization Page."""
+
+from pathlib import Path
+import pandas as pd
+import polars as pl
+import streamlit as st
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+# Import normalization engine functions from openms_insight
+from openms_insight.analysis.normalization import (
+ normalize_samples,
+ scale_data,
+ transform_data,
+)
+
+STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"]
+
+
+def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None:
+ """Keep preprocessing tables intensity-only before statistical analysis."""
+ if df is None:
+ return None
+ return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore")
+
+params = page_setup()
+st.title("Data Normalization & Scaling")
+
+st.markdown(
+ """
+Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+# Load primary database assets
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+ st.info(
+ "Abundance data not available. Please run the workflow and configure sample groups first."
+ )
+ st.page_link(
+ "content/results_abundance.py", label="Go to Abundance", icon="๐"
+ )
+ st.stop()
+
+pivot_df, expr_df, group_map = result
+pivot_df = strip_stat_columns(pivot_df)
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+filtered_df = strip_stat_columns(st.session_state.get("filtered_df"))
+imputed_df = strip_stat_columns(st.session_state.get("imputed_df"))
+normalized_df = strip_stat_columns(st.session_state.get("normalized_df"))
+if filtered_df is not None:
+ st.session_state["filtered_df"] = filtered_df
+if imputed_df is not None:
+ st.session_state["imputed_df"] = imputed_df
+if normalized_df is not None:
+ st.session_state["normalized_df"] = normalized_df
+
+# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) ---
+if (
+ "imputed_df" in st.session_state
+ and st.session_state["imputed_df"] is not None
+):
+ base_df = imputed_df
+ st.info(
+ "๐ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step."
+ )
+elif (
+ "filtered_df" in st.session_state
+ and st.session_state["filtered_df"] is not None
+):
+ base_df = filtered_df
+ st.warning(
+ "โ ๏ธ **Imputation Skipped**: Using data processed from the **Filtering** step."
+ )
+else:
+ base_df = pivot_df
+ st.warning(
+ "โ ๏ธ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table."
+ )
+
+# 2. Extract actual active sample columns dynamically
+sample_cols = [
+ c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"]
+]
+
+# --- SECTION 1: Active Input Table Preview ---
+st.subheader("Input Table Overview")
+st.markdown(
+ f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block."
+)
+st.dataframe(base_df, use_container_width=True)
+
+st.markdown("### Pipeline Overview")
+st.caption("Data flows in order: Filtering -> Imputation -> Normalization")
+
+step_rows = [
+ {
+ "Step": "Filtering",
+ "Status": "Done" if filtered_df is not None else "Not run",
+ "Rows": filtered_df.shape[0] if filtered_df is not None else "-",
+ "Cols": filtered_df.shape[1] if filtered_df is not None else "-",
+ },
+ {
+ "Step": "Imputation",
+ "Status": "Done" if imputed_df is not None else "Not run",
+ "Rows": imputed_df.shape[0] if imputed_df is not None else "-",
+ "Cols": imputed_df.shape[1] if imputed_df is not None else "-",
+ },
+ {
+ "Step": "Normalization",
+ "Status": "Done" if normalized_df is not None else "Not run",
+ "Rows": normalized_df.shape[0] if normalized_df is not None else "-",
+ "Cols": normalized_df.shape[1] if normalized_df is not None else "-",
+ },
+]
+st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True)
+
+with st.expander("Show step tables", expanded=False):
+ if filtered_df is not None:
+ st.markdown("#### Filtering output")
+ st.dataframe(filtered_df.head(10), use_container_width=True)
+ if imputed_df is not None:
+ st.markdown("#### Imputation output")
+ st.dataframe(imputed_df.head(10), use_container_width=True)
+ if normalized_df is not None:
+ st.markdown("#### Normalization output")
+ st.dataframe(normalized_df.head(10), use_container_width=True)
+ if filtered_df is None and imputed_df is None and normalized_df is None:
+ st.info("No preprocessing outputs yet. Start from Filtering.")
+
+st.markdown("---")
+
+# --- SECTION 2: Normalization Parameter Configuration ---
+st.subheader("Configure Preprocessing & Scaling Chains")
+
+# Prepare structural Polars metadata DataFrame required by backend functions
+metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map]
+metadata_pl = pl.DataFrame(
+ metadata_rows, schema={"sample_id": pl.String, "group": pl.String}
+)
+
+col1, col2, col3 = st.columns(3)
+
+with col1:
+ st.markdown("### ๐งฌ 1. Mathematical Transformation")
+ transform_strategy = st.selectbox(
+ "Select Transformation",
+ options=["None", "log2", "log10", "square_root", "cube_root"],
+ index=0,
+ help="Compress data dynamic range and stabilize heteroscedastic variance profiles.",
+ )
+
+with col2:
+ st.markdown("### ๐งช 2. Sample Normalization")
+ norm_strategy = st.selectbox(
+ "Select Normalization",
+ options=["None", "sum", "median", "pqn", "reference_feature", "quantile"],
+ index=0,
+ help="Perform column-wise corrections to account for variable sample loading concentrations.",
+ )
+
+ # Conditionally display target input field for reference feature matching
+ ref_feature_input = None
+ if norm_strategy == "reference_feature":
+ ref_feature_input = st.text_input(
+ "Reference Protein Name (ID)",
+ value="",
+ placeholder="e.g., P01234 or GAPDH",
+ help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.",
+ )
+
+with col3:
+ st.markdown("### ๐ 3. Row Scaling")
+ scaling_strategy = st.selectbox(
+ "Select Scaling Mode",
+ options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"],
+ index=0,
+ help="Adjust individual feature weights to make low and high abundance proteins comparable.",
+ )
+
+
+# --- SECTION 3: Normalization Pipe Sequential Execution ---
+st.markdown("
", unsafe_allow_html=True)
+if st.button("Apply Normalization Pipelines", type="primary"):
+
+ # Validate reference feature selection if active before hitting polars execution layers
+ if norm_strategy == "reference_feature" and not ref_feature_input:
+ st.error(
+ "โ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy."
+ )
+ st.stop()
+
+ # Convert pandas memory buffer into optimization lazy dataframe tree graph
+ processing_lazy = pl.from_pandas(base_df).lazy()
+
+ # Execute Chain 1: Transform Matrix Data
+ try:
+ processing_lazy = transform_data(
+ quantification_data=processing_lazy,
+ metadata=metadata_pl,
+ strategy=transform_strategy,
+ )
+
+ # Execute Chain 2: Normalize Sample Intensities (Columns)
+ processing_lazy = normalize_samples(
+ quantification_data=processing_lazy,
+ metadata=metadata_pl,
+ strategy=norm_strategy,
+ id_col=id_col,
+ reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None,
+ )
+
+ # Execute Chain 3: Scale Individual Features (Rows)
+ processing_lazy = scale_data(
+ quantification_data=processing_lazy,
+ metadata=metadata_pl,
+ strategy=scaling_strategy,
+ )
+
+ # Finalize and collect pipeline query graph optimizations
+ normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas())
+
+ # ๐พ Save processing checkpoint inside Session State for Downstream (Statistics Block)
+ st.session_state["normalized_df"] = normalized_df
+
+ st.success("Successfully executed all selected normalization pipelines!")
+
+ # Display the finalized transformation matrix view
+ st.subheader("Normalized Abundance Table")
+ st.dataframe(normalized_df, use_container_width=True)
+
+ except ValueError as val_err:
+ # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein)
+ st.error(f"Engine Configuration Error: {str(val_err)}")
+ except Exception as e:
+ st.error(f"An unexpected pipeline error occurred: {str(e)}")
\ No newline at end of file
diff --git a/content/results_abundance.py b/content/results_abundance.py
index a86f1a3..38c42bc 100644
--- a/content/results_abundance.py
+++ b/content/results_abundance.py
@@ -1,6 +1,7 @@
"""Abundance (ProteomicsLFQ) Results Page."""
import streamlit as st
import pandas as pd
+import numpy as np
from pathlib import Path
from src.common.common import page_setup
from src.common.results_helpers import get_workflow_dir, get_abundance_data
@@ -12,7 +13,7 @@
st.markdown(
"""
View protein and PSM-level quantification from **ProteomicsLFQ**.
-This page calculates differential expression statistics between sample groups.
+This page focuses on raw abundance intensity for preprocessing.
"""
)
@@ -40,49 +41,44 @@
csv_file = csv_files[0]
-def render_protein_table(pivot_df, group_map, is_lfq=True):
+def render_protein_table(pivot_df, is_lfq=True):
"""Common function to render the protein-level abundance table"""
+ pivot_df = pivot_df.copy()
st.markdown("### Protein-Level Abundance Table")
st.info(
"This protein-level table is generated by grouping all PSMs that map to the "
- "same protein and aggregating their intensities across samples.\n\n"
- "Additionally, log2 fold change and p-values are calculated between sample groups."
+ "same protein and aggregating their intensities across samples."
)
- # Display group comparison info
- groups = sorted(set(group_map.values()))
- if len(groups) >= 2:
- group1, group2 = sorted(groups)[:2]
- st.info(f"Statistical comparison: **{group2} vs {group1}**")
-
if is_lfq:
# Handle LFQ mode columns (Raw Intensity)
id_col = "ProteinName"
- exclude_cols = [id_col, "log2FC", "p-value", "PeptideSequence"]
+ exclude_cols = [id_col, "PeptideSequence"]
sample_cols = [c for c in pivot_df.columns if c not in exclude_cols]
-
+
pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1)
- display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"]
+ display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"]
help_text = "Raw sample intensities"
y_min = None
else:
# Handle non-LFQ mode columns (Log2-transformed Intensity)
id_col = "protein"
- exclude_cols = [id_col, "log2FC", "p-value", "p-adj", "n_proteins", "n_peptides", "protein_score"]
+ exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"]
sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()]
-
+
pivot_df["Intensity"] = pivot_df[sample_cols].apply(
lambda row: [np.log2(v + 1) for v in row], axis=1
)
- display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols
+ display_cols = [id_col, "Intensity"] + sample_cols
help_text = "Sample intensities (log2 scale)"
y_min = 0
# Filter to available columns, then sort and display
available_cols = [c for c in display_cols if c in pivot_df.columns]
-
+ view_df = pivot_df[available_cols]
+
st.dataframe(
- pivot_df[available_cols].sort_values("p-value"),
+ view_df,
column_config={
"Intensity": st.column_config.BarChartColumn(
"Intensity",
@@ -110,12 +106,11 @@ def render_protein_table(pivot_df, group_map, is_lfq=True):
with protein_tab:
if result is None:
- st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.")
- # st.page_link("content/workflow_configure.py", label="Go to Configure", icon="โ๏ธ")
+ st.warning("Could not load abundance data. Please run the workflow first.")
st.stop()
pivot_df, expr_df, group_map = result
- render_protein_table(pivot_df, group_map, is_lfq=True)
+ render_protein_table(pivot_df, is_lfq=True)
with psm_tab:
st.markdown("### PSM-level Quantification Table")
@@ -130,27 +125,27 @@ def render_protein_table(pivot_df, group_map, is_lfq=True):
pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"])
if result is None:
- st.info("๐ก Please complete the configuration in the 'Configure' page to see results.")
+ st.info("๐ก Please run the workflow first to see results.")
st.stop()
-
+
pivot_df, expr_df, group_map = result
with pre_processing_tab:
- st.write("### Final Results (Group row removed, Stats added)")
+ st.write("### Final Results (Intensity matrix)")
st.dataframe(pivot_df.head(10))
with protein_tab:
- render_protein_table(pivot_df, group_map, is_lfq=False)
+ render_protein_table(pivot_df, is_lfq=False)
except Exception as e:
st.error(f"Failed to load {csv_file.name}: {e}")
st.markdown("---")
-st.markdown("**Next steps:** Explore statistical visualizations")
+st.markdown("**Next steps:** Continue preprocessing, then run statistical inference")
col1, col2, col3 = st.columns(3)
with col1:
- st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
+ st.page_link("content/filtering.py", label="Filtering", icon="๐งน")
with col2:
- st.page_link("content/results_pca.py", label="PCA", icon="๐")
+ st.page_link("content/imputation.py", label="Imputation", icon="๐งฉ")
with col3:
- st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
+ st.page_link("content/statistical.py", label="Statistical Inference", icon="๐ฌ")
\ No newline at end of file
diff --git a/content/results_heatmap.py b/content/results_heatmap.py
index 72b8438..104bff6 100644
--- a/content/results_heatmap.py
+++ b/content/results_heatmap.py
@@ -1,20 +1,18 @@
"""Heatmap Results Page."""
import streamlit as st
import numpy as np
-import plotly.express as px
-from scipy.cluster.hierarchy import linkage, leaves_list
-from scipy.spatial.distance import pdist
+import polars as pl
from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_workflow_dir
-from src.workflow.ParameterManager import ParameterManager
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+from openms_insight import Heatmap
params = page_setup()
st.title("Heatmap")
st.markdown(
"""
-Hierarchically clustered heatmap of protein-level abundance (Z-score normalized).
-Proteins and samples are ordered by similarity.
+Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized).
+Powered by OpenMS-Insight multi-resolution engine.
"""
)
@@ -29,104 +27,75 @@
st.stop()
pivot_df, expr_df, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
-workflow_dir = get_workflow_dir(st.session_state["workspace"])
-parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow")
-
-workflow_params = parameter_manager.get_parameters_from_json()
-analysis_mode = workflow_params.get("analysis-mode", "LFQ")
-
-st.write("Workflow Analysis Mode:", analysis_mode)
-
-if analysis_mode == "LFQ":
- top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n")
-
- var_series = expr_df.var(axis=1)
- top_proteins = var_series.sort_values(ascending=False).head(top_n).index
- heatmap_df = expr_df.loc[top_proteins]
- heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0)
- heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna()
-
- if not heatmap_z.empty:
- row_linkage = linkage(pdist(heatmap_z.values), method="average")
- row_order = leaves_list(row_linkage)
-
- col_linkage = linkage(pdist(heatmap_z.T.values), method="average")
- col_order = leaves_list(col_linkage)
-
- heatmap_clustered = heatmap_z.iloc[row_order, col_order]
-
- fig_heatmap = px.imshow(
- heatmap_clustered,
- labels=dict(x="Sample", y="Protein", color="Z-score"),
- aspect="auto",
- color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]],
- zmin=-3, zmax=3
- )
-
- fig_heatmap.update_layout(
- height=700,
- xaxis={'side': 'bottom'},
- yaxis={'side': 'left'}
- )
-
- fig_heatmap.update_xaxes(tickfont=dict(size=10))
- fig_heatmap.update_yaxes(tickfont=dict(size=8))
-
- st.plotly_chart(fig_heatmap, use_container_width=True)
- else:
- st.warning("Insufficient data to generate the heatmap.")
+if expr_df.empty:
+ st.info("No data available for heatmap.")
+ st.stop()
- st.markdown("---")
- st.markdown("**Other visualizations:**")
- col1, col2 = st.columns(2)
- with col1:
- st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
- with col2:
- st.page_link("content/results_pca.py", label="PCA", icon="๐")
+sample_cols = expr_df.columns.tolist()
+
+# UI settings (number of top variance proteins)
+top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n")
+
+# Process data (variance selection -> Z-score normalization)
+var_series = expr_df.var(axis=1)
+top_proteins = var_series.sort_values(ascending=False).head(top_n).index
+heatmap_df = expr_df.loc[top_proteins]
+
+# Compute Z-scores and clean missing/invalid values
+heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0)
+heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna()
+
+if not heatmap_z.empty:
+ # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements
+ # Restore the id column from the index as a regular column
+ heatmap_z_reset = heatmap_z.reset_index()
+
+ # Unpivot the wide-format matrix into long-format (X, Y, Intensity)
+ melted_df = heatmap_z_reset.melt(
+ id_vars=[id_col],
+ value_vars=sample_cols,
+ var_name="Sample",
+ value_name="Z_score"
+ )
+
+ # Add sample group mapping if available for heatmap categories
+ if sample_group_map:
+ melted_df["Group"] = melted_df["Sample"].map(sample_group_map)
+
+ # Pack the Pandas DataFrame into a Polars LazyFrame
+ heatmap_pl_lazy = pl.from_pandas(melted_df).lazy()
+
+ # Initialize the OpenMS-Insight Heatmap component and map attributes
+ heatmap_component = Heatmap(
+ cache_id="quantms_protein_heatmap",
+ x_column="Sample",
+ y_column=id_col,
+ data=heatmap_pl_lazy,
+ intensity_column="Z_score",
+ title="Protein Abundance Heatmap (Z-score)",
+ x_label="Samples",
+ y_label="Proteins",
+ colorscale="RdBu",
+ reversescale=True,
+ log_scale=False, # Z-score can be negative, so log scale must stay off
+ intensity_label="Z-score",
+ category_column=None,
+ min_points=10000, # Generous point-count ceiling so the full grid renders
+ )
+
+ # Render the component
+ state_manager = st.session_state.get("state")
+ heatmap_component(state_manager=state_manager)
else:
- top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n")
-
- var_series = expr_df.var(axis=1)
- top_proteins = var_series.sort_values(ascending=False).head(top_n).index
- heatmap_df = expr_df.loc[top_proteins]
- heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0)
- heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna()
-
- if not heatmap_z.empty:
- row_linkage = linkage(pdist(heatmap_z.values), method="average")
- row_order = leaves_list(row_linkage)
-
- col_linkage = linkage(pdist(heatmap_z.T.values), method="average")
- col_order = leaves_list(col_linkage)
-
- heatmap_clustered = heatmap_z.iloc[row_order, col_order]
-
- fig_heatmap = px.imshow(
- heatmap_clustered,
- labels=dict(x="Sample", y="Protein", color="Z-score"),
- aspect="auto",
- color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]],
- zmin=-3, zmax=3
- )
-
- fig_heatmap.update_layout(
- height=700,
- xaxis={'side': 'bottom'},
- yaxis={'side': 'left'}
- )
-
- fig_heatmap.update_xaxes(tickfont=dict(size=10))
- fig_heatmap.update_yaxes(tickfont=dict(size=8))
-
- st.plotly_chart(fig_heatmap, width="stretch")
- else:
- st.warning("Insufficient data to generate the heatmap.")
-
- st.markdown("---")
- st.markdown("**Other visualizations:**")
- col1, col2 = st.columns(2)
- with col1:
- st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
- with col2:
- st.page_link("content/results_pca.py", label="PCA", icon="๐")
+ st.warning("Insufficient data to generate the heatmap.")
+
+st.markdown("---")
+st.markdown("**Other visualizations:**")
+col1, col2 = st.columns(2)
+with col1:
+ st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
+with col2:
+ st.page_link("content/results_pca.py", label="PCA", icon="๐")
diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py
new file mode 100644
index 0000000..7104c3a
--- /dev/null
+++ b/content/results_heatmap_clustered.py
@@ -0,0 +1,107 @@
+"""Clustered Heatmap Results Page."""
+import streamlit as st
+import numpy as np
+import polars as pl
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+from openms_insight import ClusteredHeatmap
+
+params = page_setup()
+st.title("Clustered Heatmap")
+
+st.markdown(
+ """
+A real grid heatmap (rows = proteins, columns = samples) with hierarchical
+clustering dendrograms on both axes and a sample-group color bar, powered
+by OpenMS-Insight.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+ st.info("Abundance data not available. Please run the workflow and configure sample groups first.")
+ st.page_link("content/results_abundance.py", label="Go to Abundance", icon="๐")
+ st.stop()
+
+pivot_df, expr_df, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+if expr_df.empty:
+ st.info("No data available for heatmap.")
+ st.stop()
+
+top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n")
+
+var_series = expr_df.var(axis=1)
+top_proteins = var_series.sort_values(ascending=False).head(top_n).index
+heatmap_df = expr_df.loc[top_proteins]
+
+heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0)
+heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna()
+
+if heatmap_z.empty:
+ st.warning("Insufficient data to generate the heatmap.")
+ st.stop()
+
+heatmap_z_reset = heatmap_z.reset_index()
+heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy()
+
+sample_cols = expr_df.columns.tolist()
+metadata_pl = pl.DataFrame(
+ [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map],
+ schema={"sample_id": pl.String, "group": pl.String},
+)
+
+# Assign group annotation-bar colors in sorted-group order (matching how
+# ClusteredHeatmap._preprocess() orders unique groups internally).
+group_palette = [
+ "#00BFC4", # teal
+ "#F8766D", # salmon
+ "#7CAE00", # yellow-green
+ "#C77CFF", # lavender purple
+ "#E7B800", # gold/amber
+ "#619CFF", # blue
+ "#FF61C3", # pink/magenta
+ "#00BA38", # green
+ "#FF8C42", # orange
+ "#00B0F6", # sky blue
+]
+unique_groups = sorted(set(sample_group_map.values()))
+group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)}
+
+heatmap_component = ClusteredHeatmap(
+ cache_id="quantms_clustered_heatmap",
+ cache_path=str(st.session_state["workspace"]),
+ id_col=id_col,
+ data=heatmap_lazy,
+ metadata=metadata_pl,
+ row_cluster=True,
+ col_cluster=True,
+ title="Protein Abundance Heatmap (Z-score, clustered)",
+ x_label="Samples",
+ y_label="Proteins",
+ colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]],
+ reversescale=False,
+ intensity_label="Z-score",
+ group_colors=group_colors,
+)
+
+state_manager = st.session_state.get("state")
+# Scale height with the number of proteins so row labels stay readable -
+# BaseComponent otherwise defaults to a flat 400px, too short for a
+# dendrogram+heatmap composite with more than a handful of rows.
+heatmap_height = max(600, min(1400, 300 + top_n * 20))
+heatmap_component(state_manager=state_manager, height=heatmap_height)
+
+st.markdown("---")
+st.markdown("**Other visualizations:**")
+col1, col2 = st.columns(2)
+with col1:
+ st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
+with col2:
+ st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="๐ฅ")
diff --git a/content/results_pca.py b/content/results_pca.py
index 466e475..29f441a 100644
--- a/content/results_pca.py
+++ b/content/results_pca.py
@@ -1,12 +1,10 @@
"""PCA Results Page."""
-import streamlit as st
import pandas as pd
-import plotly.express as px
-from sklearn.decomposition import PCA
-from sklearn.preprocessing import StandardScaler
+import polars as pl
+import streamlit as st
from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_workflow_dir
-from src.workflow.ParameterManager import ParameterManager
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+from openms_insight import PCAPlot
params = page_setup()
st.title("PCA Analysis")
@@ -14,7 +12,7 @@
st.markdown(
"""
Principal Component Analysis (PCA) of protein-level abundance.
-Samples are colored by group assignment to visualize clustering.
+Samples are projected onto their principal components and colored by group assignment to visualize clustering.
"""
)
@@ -22,6 +20,7 @@
st.warning("Please initialize your workspace first.")
st.stop()
+# 1. Load abundance data (base wide-format table + sample -> group mapping)
result = get_abundance_data(st.session_state["workspace"])
if result is None:
st.info("Abundance data not available. Please run the workflow and configure sample groups first.")
@@ -29,88 +28,143 @@
st.stop()
pivot_df, expr_df, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) ---
+# Mirrors statistical.py: PCA should run on the most-processed data available.
+if (
+ "normalized_df" in st.session_state
+ and st.session_state["normalized_df"] is not None
+):
+ base_df = st.session_state["normalized_df"]
+ st.info(
+ "๐ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step."
+ )
+elif (
+ "imputed_df" in st.session_state
+ and st.session_state["imputed_df"] is not None
+):
+ base_df = st.session_state["imputed_df"]
+ st.warning(
+ "โ ๏ธ **Normalization Skipped**: Using data processed from the **Imputation** step."
+ )
+elif (
+ "filtered_df" in st.session_state
+ and st.session_state["filtered_df"] is not None
+):
+ base_df = st.session_state["filtered_df"]
+ st.warning(
+ "โ ๏ธ **Preprocessing Skipped**: Using data processed from the **Filtering** step."
+ )
+else:
+ base_df = pivot_df
+ st.warning(
+ "โ ๏ธ **Raw Input Active**: No preprocessing history found. Operating on the original table."
+ )
+
+# 2. Extract active sample columns and detect unique biological groups
+sample_cols = [
+ c for c in base_df.columns
+ if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"]
+]
+unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map})
-workflow_dir = get_workflow_dir(st.session_state["workspace"])
-parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow")
-workflow_params = parameter_manager.get_parameters_from_json()
-analysis_mode = workflow_params.get("analysis-mode", "LFQ")
+if len(sample_cols) < 2:
+ st.info("PCA requires at least 2 samples.")
+ st.stop()
-st.write("Workflow Analysis Mode:", analysis_mode)
+if len(unique_groups) < 2:
+ st.warning(
+ "Only one biological group was detected - points will still be plotted, "
+ "but group-based coloring requires 2 or more groups."
+ )
-top_n = 500
+# --- SECTION 1: Active Input Table Preview ---
+st.subheader("Input Table Overview")
+st.markdown(
+ f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples "
+ f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})."
+)
+st.dataframe(base_df, use_container_width=True)
-if analysis_mode == "LFQ":
- protein_col = "ProteinName"
-else:
- protein_col = "protein"
+st.markdown("---")
-top_proteins = (
- pivot_df
- .dropna(subset=["p-adj"])
- .sort_values("p-adj", ascending=True)
- .head(top_n)[protein_col]
-)
+# --- SECTION 2: PCA Configuration ---
+st.subheader("Configure PCA")
-expr_df_pca = expr_df.loc[
- expr_df.index.intersection(top_proteins)
-]
+expr_df_wide = base_df.set_index(id_col)[sample_cols]
+max_available = expr_df_wide.shape[0]
+
+if max_available <= 20:
+ top_n = max_available
+ st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).")
+else:
+ top_n = st.slider(
+ "Number of proteins (Highest Variance)",
+ min_value=20,
+ max_value=min(5000, max_available),
+ value=min(500, max_available),
+ step=10,
+ key="pca_top_n",
+ help=(
+ "PCA is computed only on the N proteins with the highest variance "
+ "across samples, to reduce noise from low-variance/uninformative features."
+ ),
+ )
+
+top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index
+expr_df_pca = expr_df_wide.loc[top_proteins].reset_index()
if expr_df_pca.shape[0] < 2:
- st.info("Not enough proteins after p-value filtering for PCA.")
+ st.info("Not enough proteins after variance filtering for PCA.")
st.stop()
-X = expr_df_pca.T
-X_scaled = StandardScaler().fit_transform(X)
-
-pca = PCA(n_components=2)
-pcs = pca.fit_transform(X_scaled)
-
-pca_df = pd.DataFrame(
- pcs,
- columns=["PC1", "PC2"],
- index=X.index
+# Prepare structural Polars metadata DataFrame required by PCAPlot
+metadata_pl = pl.DataFrame(
+ [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map],
+ schema={"sample_id": pl.String, "group": pl.String},
)
+pca_lazy = pl.from_pandas(expr_df_pca).lazy()
+
+# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally)
+try:
+ pca_component = PCAPlot(
+ cache_id="quantms_pca_plot",
+ data=pca_lazy,
+ metadata=metadata_pl,
+ sample_id_field="sample_id",
+ group_field="group",
+ n_components=5,
+ title="Sample PCA",
+ )
+except ValueError as e:
+ st.error(f"PCA computation failed: {e}")
+ st.stop()
-if analysis_mode == "LFQ":
- norm_map = {
- k.replace(".mzML", ""): v
- for k, v in group_map.items()
- }
-else:
- actual_sample_names = pca_df.index.tolist()
- norm_map = {}
- for k, v in group_map.items():
- try:
- sample_idx = int(k) + 1
- target_substring = f"sample{sample_idx}["
- real_full_name = next((name for name in actual_sample_names if target_substring in name), None)
-
- if real_full_name:
- norm_map[real_full_name] = v if v and v.strip() else "Unassigned"
- except ValueError:
- continue
-
-pca_df["Group"] = pca_df.index.map(norm_map)
-
-fig_pca = px.scatter(
- pca_df,
- x="PC1",
- y="PC2",
- color="Group",
- text=pca_df.index,
-)
+variance_ratio = pca_component.get_variance_ratio()
+pc_columns = pca_component.get_pc_columns()
-fig_pca.update_traces(textposition="top center")
-fig_pca.update_layout(
- xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)",
- yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)",
- height=600,
-)
+# 4. Let the user pick which component pair to view (no recomputation needed)
+col1, col2 = st.columns(2)
+with col1:
+ pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x")
+with col2:
+ default_y_index = 1 if len(pc_columns) > 1 else 0
+ pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y")
-st.plotly_chart(fig_pca, width="stretch")
+pc_x = int(pc_x_label.replace("PC", ""))
+pc_y = int(pc_y_label.replace("PC", ""))
-st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)")
-st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)")
+# 5. Render the component
+state_manager = st.session_state.get("state")
+pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600)
+
+st.markdown(
+ "**Explained variance:** "
+ + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio))
+)
+st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)")
st.markdown("---")
st.markdown("**Other visualizations:**")
@@ -118,4 +172,4 @@
with col1:
st.page_link("content/results_volcano.py", label="Volcano Plot", icon="๐")
with col2:
- st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
\ No newline at end of file
+ st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py
index 77eb332..fde2ab9 100644
--- a/content/results_proteomicslfq.py
+++ b/content/results_proteomicslfq.py
@@ -45,15 +45,14 @@
st.markdown("### ๐งฌ Protein-Level Abundance Table")
st.info(
"This protein-level table is generated by grouping all PSMs that map to the "
- "same protein and aggregating their intensities across samples.\n\n"
- "Additionally, log2 fold change and p-values are calculated between sample groups."
+ "same protein and aggregating their intensities across samples."
)
if pivot_df.empty:
st.info("No protein-level data available.")
else:
st.session_state["pivot_df"] = pivot_df
- st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True)
+ st.dataframe(pivot_df, use_container_width=True)
# ======================================================
# GO Enrichment Results
diff --git a/content/results_volcano.py b/content/results_volcano.py
index 5cc9b29..db2702f 100644
--- a/content/results_volcano.py
+++ b/content/results_volcano.py
@@ -1,10 +1,9 @@
"""Volcano Plot Results Page."""
import streamlit as st
-import plotly.express as px
-import numpy as np
+import polars as pl
from src.common.common import page_setup
-from src.common.results_helpers import get_abundance_data, get_workflow_dir
-from src.workflow.ParameterManager import ParameterManager
+from src.common.results_helpers import get_abundance_data, get_id_column
+from openms_insight import VolcanoPlot
params = page_setup()
st.title("Volcano Plot")
@@ -20,6 +19,19 @@
st.warning("Please initialize your workspace first.")
st.stop()
+# 1. Check if statistical analysis results are available in the session state
+if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None:
+ st.info("Statistical analysis data not found. Please run the statistical engine first.")
+ st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="๐ฌ")
+ st.stop()
+
+# Retrieve the completed statistical analysis DataFrame
+statistics_df = st.session_state["statistics_df"]
+
+if statistics_df.empty:
+ st.info("No data available for volcano plot.")
+ st.stop()
+
result = get_abundance_data(st.session_state["workspace"])
if result is None:
st.info("Abundance data not available. Please run the workflow and configure sample groups first.")
@@ -27,166 +39,63 @@
st.stop()
pivot_df, expr_df, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+
+# 2. Clean data and convert to Polars for component input
+volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy()
+volcano_pl_lazy = pl.from_pandas(volcano_df).lazy()
+
+# 3. Configure UI sliders (changing thresholds does not invalidate cache)
+fc_thresh = st.slider(
+ "log2 Fold Change threshold",
+ min_value=0.5,
+ max_value=3.0,
+ value=1.0,
+ step=0.1,
+)
+
+p_thresh = st.slider(
+ "p-adj (FDR) threshold",
+ min_value=0.001,
+ max_value=0.1,
+ value=0.05,
+ step=0.001,
+)
+
+# 4. Initialize the OpenMS-Insight VolcanoPlot component
+volcano_plot_component = VolcanoPlot(
+ cache_id="quantms_volcano_plot",
+ data=volcano_pl_lazy,
+ log2fc_column="log2FC",
+ pvalue_column="p-adj",
+ label_column=id_col,
+ up_color="#E74C3C",
+ down_color="#3498DB",
+ ns_color="#95A5A6",
+ show_threshold_lines=True,
+ threshold_line_style="dash",
+)
+
+# 5. Render the component
+state_manager = st.session_state.get("state") # Inject the project state management object
+
+volcano_plot_component(
+ state_manager=state_manager,
+ fc_threshold=fc_thresh,
+ p_threshold=p_thresh,
+ max_labels=10, # Display labels for the top N significant proteins
+ height=600,
+)
-if pivot_df.empty:
- st.info("No data available for volcano plot.")
- st.stop()
-
-workflow_dir = get_workflow_dir(st.session_state["workspace"])
-parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow")
-
-workflow_params = parameter_manager.get_parameters_from_json()
-analysis_mode = workflow_params.get("analysis-mode", "LFQ")
-
-st.write("Workflow Analysis Mode:", analysis_mode)
-
-if analysis_mode == "LFQ":
- volcano_df = pivot_df.copy()
- volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"])
-
- volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"])
-
- fc_thresh = st.slider(
- "log2 Fold Change threshold",
- min_value=0.5,
- max_value=3.0,
- value=1.0,
- step=0.1,
- )
-
- p_thresh = st.slider(
- "p-adj (FDR) threshold",
- min_value=0.001,
- max_value=0.1,
- value=0.05,
- step=0.001,
- )
-
- volcano_df["Significance"] = "Not significant"
- volcano_df.loc[
- (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh),
- "Significance",
- ] = "Up-regulated"
-
- volcano_df.loc[
- (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh),
- "Significance",
- ] = "Down-regulated"
-
- fig_volcano = px.scatter(
- volcano_df,
- x="log2FC",
- y="neg_log10_padj",
- color="Significance",
- hover_data=["ProteinName", "log2FC", "p-value", "p-adj"],
- color_discrete_map={
- "Up-regulated": "red",
- "Down-regulated": "blue",
- "Not significant": "lightgrey",
- }
- )
-
- fig_volcano.add_vline(x=fc_thresh, line_dash="dash")
- fig_volcano.add_vline(x=-fc_thresh, line_dash="dash")
- fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash")
-
- # Make x-axis symmetric around zero
- max_abs_fc = volcano_df["log2FC"].abs().max()
- x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding
-
- fig_volcano.update_layout(
- xaxis_title="log2 Fold Change",
- yaxis_title="-log10(p-adj)",
- xaxis_range=x_range,
- height=600,
- )
-
- st.plotly_chart(fig_volcano, use_container_width=True)
-
- up_count = (volcano_df["Significance"] == "Up-regulated").sum()
- down_count = (volcano_df["Significance"] == "Down-regulated").sum()
- st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}")
-
- st.markdown("---")
- st.markdown("**Other visualizations:**")
- col1, col2 = st.columns(2)
- with col1:
- st.page_link("content/results_pca.py", label="PCA", icon="๐")
- with col2:
- st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
-else:
- # Threshold Selection UI
- st.divider()
- c1, c2 = st.columns(2)
- with c1:
- fc_thresh = st.slider(
- "log2 Fold Change threshold",
- min_value=0.1,
- max_value=3.0,
- value=1.0,
- step=0.1,
- )
- with c2:
- p_thresh = st.slider(
- "p-adj (FDR) threshold",
- min_value=0.001,
- max_value=0.1,
- value=0.05,
- step=0.001,
- )
-
- volcano_df = pivot_df.dropna(subset=["log2FC", "p-adj"]).copy()
- volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"])
-
- volcano_df["Significance"] = "Not significant"
- volcano_df.loc[
- (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh),
- "Significance",
- ] = "Up-regulated"
-
- volcano_df.loc[
- (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh),
- "Significance",
- ] = "Down-regulated"
-
- fig_volcano = px.scatter(
- volcano_df,
- x="log2FC",
- y="neg_log10_padj",
- color="Significance",
- hover_data=["protein", "log2FC", "p-value", "p-adj"],
- color_discrete_map={
- "Up-regulated": "red",
- "Down-regulated": "blue",
- "Not significant": "lightgrey",
- }
- )
-
- fig_volcano.add_vline(x=fc_thresh, line_dash="dash")
- fig_volcano.add_vline(x=-fc_thresh, line_dash="dash")
- fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash")
-
- # Make x-axis symmetric around zero
- max_abs_fc = volcano_df["log2FC"].abs().max()
- x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding
-
- fig_volcano.update_layout(
- xaxis_title="log2 Fold Change",
- yaxis_title="-log10(p-adj)",
- xaxis_range=x_range,
- height=600,
- )
-
- st.plotly_chart(fig_volcano, width="stretch")
-
- up_count = (volcano_df["Significance"] == "Up-regulated").sum()
- down_count = (volcano_df["Significance"] == "Down-regulated").sum()
- st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}")
-
- st.markdown("---")
- st.markdown("**Other visualizations:**")
- col1, col2 = st.columns(2)
- with col1:
- st.page_link("content/results_pca.py", label="PCA", icon="๐")
- with col2:
- st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
+# 6. Keep the existing statistical summary and bottom links
+up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum()
+down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum()
+st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}")
+
+st.markdown("---")
+st.markdown("**Other visualizations:**")
+col1, col2 = st.columns(2)
+with col1:
+ st.page_link("content/results_pca.py", label="PCA", icon="๐")
+with col2:
+ st.page_link("content/results_heatmap.py", label="Heatmap", icon="๐ฅ")
diff --git a/content/statistical.py b/content/statistical.py
new file mode 100644
index 0000000..2e2a46d
--- /dev/null
+++ b/content/statistical.py
@@ -0,0 +1,165 @@
+"""Statistical Inference Page."""
+
+from pathlib import Path
+import pandas as pd
+import polars as pl
+import streamlit as st
+from src.common.common import page_setup
+from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map
+# Import statistics engine functions from openms_insight
+from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy
+
+params = page_setup()
+st.title("Statistical Inference")
+
+st.markdown(
+ """
+Run differential expression analysis to identify statistically significant proteins across your biological groups.
+"""
+)
+
+if "workspace" not in st.session_state:
+ st.warning("Please initialize your workspace first.")
+ st.stop()
+
+# Load primary database assets
+result = get_abundance_data(st.session_state["workspace"])
+if result is None:
+ st.info(
+ "Abundance data not available. Please run the workflow and configure sample groups first."
+ )
+ st.page_link(
+ "content/results_abundance.py", label="Go to Abundance", icon="๐"
+ )
+ st.stop()
+
+pivot_df, expr_df, group_map = result
+id_col = get_id_column(st.session_state["workspace"], pivot_df)
+sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map)
+
+# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) ---
+if (
+ "normalized_df" in st.session_state
+ and st.session_state["normalized_df"] is not None
+):
+ base_df = st.session_state["normalized_df"]
+ st.info(
+ "๐ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step."
+ )
+elif (
+ "imputed_df" in st.session_state
+ and st.session_state["imputed_df"] is not None
+):
+ base_df = st.session_state["imputed_df"]
+ st.warning(
+ "โ ๏ธ **Normalization Skipped**: Using data processed from the **Imputation** step."
+ )
+elif (
+ "filtered_df" in st.session_state
+ and st.session_state["filtered_df"] is not None
+):
+ base_df = st.session_state["filtered_df"]
+ st.warning(
+ "โ ๏ธ **Preprocessing Skipped**: Using data processed from the **Filtering** step."
+ )
+else:
+ base_df = pivot_df
+ st.warning(
+ "โ ๏ธ **Raw Input Active**: No preprocessing history found. Operating on the original table."
+ )
+
+# 2. Extract actual active sample columns and detect unique biological groups
+sample_cols = [
+ c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"]
+]
+unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map])))
+group_count = len(unique_groups)
+
+# --- SECTION 1: Active Input Table Preview ---
+st.subheader("Input Table Overview")
+st.markdown(
+ f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})."
+)
+st.dataframe(base_df, use_container_width=True)
+
+st.markdown("---")
+
+# --- SECTION 2: Dynamic Statistical Parameter Configuration ---
+st.subheader("Configure Statistical Engine")
+
+# Prepare structural Polars metadata DataFrame required by backend functions
+metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map]
+metadata_pl = pl.DataFrame(
+ metadata_rows, schema={"sample_id": pl.String, "group": pl.String}
+)
+
+col1, col2 = st.columns(2)
+
+with col1:
+ st.markdown("### ๐ฌ 1. Hypothesis Testing Method")
+
+ # Route available method options dynamically based on the group count
+ if group_count == 2:
+ method_options = ["limma_like", "welch", "paired"]
+ help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples."
+ elif group_count >= 3:
+ method_options = ["limma_like", "anova"]
+ help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA."
+ else:
+ st.error("โ Statistical testing requires at least 2 unique sample groups.")
+ st.stop()
+
+ selected_method = st.selectbox(
+ "Select Statistical Test",
+ options=method_options,
+ index=0,
+ help=help_text
+ )
+
+with col2:
+ st.markdown("### ๐ก๏ธ 2. Multiple Testing Correction (FDR)")
+ selected_fdr = st.selectbox(
+ "Select FDR Adjustment Strategy",
+ options=["BH", "Bonferroni", "None"],
+ index=0,
+ help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control."
+ )
+
+# --- SECTION 3: Statistical Query Execution ---
+st.markdown("
", unsafe_allow_html=True)
+if st.button("Run Statistical Analysis", type="primary"):
+
+ # Convert active pandas dataframe into polars lazyframe graph
+ stats_lazy = pl.from_pandas(base_df).lazy()
+
+ try:
+ # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value)
+ stats_lazy = calculate_statistical_tests(
+ quantification_data=stats_lazy,
+ metadata=metadata_pl,
+ method=selected_method
+ )
+
+ # Execute Chain 2: Adjust Multiple Testing (Adds p-adj)
+ stats_lazy = adjust_fdr_lazy(
+ quantification_data=stats_lazy,
+ strategy=selected_fdr
+ )
+
+ # Resolve lazy graph optimization tree and bring back to pandas memory
+ statistics_df = stats_lazy.collect().to_pandas()
+
+ # ๐พ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI)
+ st.session_state["statistics_df"] = statistics_df
+
+ st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!")
+
+ # Display the finalized statistics table view
+ st.subheader("Statistical Analysis Results")
+ st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`")
+ st.dataframe(statistics_df, use_container_width=True)
+
+ except ValueError as val_err:
+ st.error(f"Engine Validation Fallure: {str(val_err)}")
+ except Exception as e:
+ st.error(f"An unexpected pipeline error occurred: {str(e)}")
\ No newline at end of file
diff --git a/pr-397.patch b/pr-397.patch
new file mode 100644
index 0000000..bb97819
--- /dev/null
+++ b/pr-397.patch
@@ -0,0 +1,119 @@
+From 268348f6da9809750e3116535f4f3fe9defc7ab1 Mon Sep 17 00:00:00 2001
+From: Yoo HoJun
+Date: Wed, 15 Jul 2026 15:04:48 +0900
+Subject: [PATCH] Add support for boolean CLI flag parameters in TOPP tools
+
+Allows input_TOPP() to designate parameters as flags (present/absent)
+instead of key-value pairs, persisted to params.json and session_state
+so run_topp() can correctly build the command line.
+---
+ src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++----------
+ src/workflow/StreamlitUI.py | 13 ++++++++
+ 2 files changed, 51 insertions(+), 15 deletions(-)
+
+diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py
+index 11bb1486..042c5e11 100644
+--- a/src/workflow/CommandExecutor.py
++++ b/src/workflow/CommandExecutor.py
+@@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool
+
+ # Load merged parameters (_defaults + user overrides) for this tool instance
+ merged_params = self.parameter_manager.get_merged_params(params_key)
++
++ # Load flag parameter names: params.json takes priority (survives session restart),
++ # session_state is the live fallback during the current session.
++ flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {})
++ if not flag_map:
++ flag_map = st.session_state.get("_topp_flag_params", {})
++ flag_params: set = set(flag_map.get(params_key, []))
++
+ # Construct commands for each process
+ for i in range(n_processes):
+ command = [tool]
+@@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool
+ command += [value[i]]
+ # Add merged TOPP tool parameters (_defaults + user overrides)
+ for k, v in merged_params.items():
+- command += [f"-{k}"]
+- # Skip only empty strings (pass flag with no value)
+- # Note: 0 and 0.0 are valid values, so use explicit check
+- if v != "" and v is not None:
+- if isinstance(v, str) and "\n" in v:
+- command += v.split("\n")
++ if k in flag_params:
++ # CLI flag: include "-k" only when truthy, omit when false
++ if isinstance(v, str):
++ is_enabled = v.lower() == "true"
+ else:
+- command += [str(v)]
++ is_enabled = bool(v)
++ if is_enabled:
++ command += [f"-{k}"]
++ continue
++ # Regular parameter: skip empty/None, append value otherwise
++ if v == "" or v is None:
++ continue
++ command += [f"-{k}"]
++ if isinstance(v, str) and "\n" in v:
++ command += v.split("\n")
++ else:
++ command += [str(v)]
+ # Add custom parameters
+ for k, v in custom_params.items():
+- command += [f"-{k}"]
+-
+- # Skip only empty strings (pass flag with no value)
+- # Note: 0 and 0.0 are valid values, so use explicit check
+- if v != "" and v is not None:
+- if isinstance(v, list):
+- command += [str(x) for x in v]
++ if k in flag_params:
++ if isinstance(v, str):
++ is_enabled = v.lower() == "true"
+ else:
+- command += [str(v)]
++ is_enabled = bool(v)
++ if is_enabled:
++ command += [f"-{k}"]
++ continue
++ if v == "" or v is None:
++ continue
++ command += [f"-{k}"]
++ if isinstance(v, list):
++ command += [str(x) for x in v]
++ else:
++ command += [str(v)]
+ # Add threads parameter for TOPP tools
+ command += ["-threads", str(threads_per_command)]
+ commands.append(command)
+diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py
+index 9c24dc2c..4bd53b18 100644
+--- a/src/workflow/StreamlitUI.py
++++ b/src/workflow/StreamlitUI.py
+@@ -826,6 +826,7 @@ def input_TOPP(
+ num_cols: int = 4,
+ exclude_parameters: List[str] = [],
+ include_parameters: List[str] = [],
++ flag_parameters: List[str] = [],
+ display_tool_name: bool = True,
+ display_subsections: bool = True,
+ display_subsection_tabs: bool = False,
+@@ -862,6 +863,18 @@ def input_TOPP(
+ st.session_state["_topp_tool_instance_map"] = {}
+ st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name
+
++ # Persist flag_parameters to session_state and params.json so run_topp
++ # can skip appending a value for these boolean CLI flags.
++ if "_topp_flag_params" not in st.session_state:
++ st.session_state["_topp_flag_params"] = {}
++ st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters)
++ _fp = self.parameter_manager.get_parameters_from_json()
++ if "_flag_params" not in _fp:
++ _fp["_flag_params"] = {}
++ _fp["_flag_params"][tool_instance_name] = list(flag_parameters)
++ with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f:
++ json.dump(_fp, _f, indent=4)
++
+ if not display_subsections:
+ display_subsection_tabs = False
+ if display_subsection_tabs:
diff --git a/requirements.txt b/requirements.txt
index aac2879..5d20693 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -142,6 +142,11 @@ scipy
scikit-learn
openms-insight>=0.1.13
polars>=1.0.0
+# Forces polars to prefer its most CPU-compatible native runtime.
+# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32)
+# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some
+# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops.
+polars-runtime-compat
cython
easypqp>=0.1.34
pyprophet>=2.2.0
@@ -149,4 +154,5 @@ mygene
# Redis Queue dependencies (for online mode)
redis>=5.0.0
rq>=1.16.0
-statsmodels
\ No newline at end of file
+statsmodels
+polars
\ No newline at end of file
diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py
index ebf7e65..120efb0 100644
--- a/src/WorkflowTest.py
+++ b/src/WorkflowTest.py
@@ -1,5 +1,6 @@
import streamlit as st
from pathlib import Path
+import re
import pandas as pd
import plotly.express as px
from streamlit_plotly_events import plotly_events
@@ -409,6 +410,7 @@ def render_tmt_tabs(self):
"quantification:isotope_correction": "false",
},
tool_instance_name="IsobaricAnalyzer-TMT",
+ reactive=True,
)
with t[1]:
comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications",
@@ -559,64 +561,62 @@ def render_tmt_tabs(self):
)
with t[10]:
st.markdown("### ๐งช TMT Sample Group Assignment")
-
- # 1. Determine TMT type (e.g., tmt10plex, tmt16plex)
- target_key = f"{self.parameter_manager.topp_param_prefix}IsobaricAnalyzer:1:type"
- selected_tmt = st.session_state.get(target_key, "tmt12plex")
-
- if "tmt" in selected_tmt:
- import re
- # Extract the number to determine the plex count
- num_plex_match = re.search(r'\d+', selected_tmt)
- if num_plex_match:
- num_plex = int(num_plex_match.group())
- all_channels = [f"sample{i+1}" for i in range(num_plex)]
-
- st.info(
- "Enter a group name for each TMT channel.\n\n"
- "Type **'skip'** for channels you wish to skip. (e.g., control, case, skip)"
- )
- # 2. Create an input_widget for each channel (automatically saved to params.json)
- cols = st.columns(2)
- for i, ch in enumerate(all_channels):
- with cols[i % 2]:
+ latest_params = self.parameter_manager.get_parameters_from_json()
+ type_key = (
+ f"{self.parameter_manager.topp_param_prefix}"
+ "IsobaricAnalyzer-TMT:1:type"
+ )
+ selected_type = str(
+ st.session_state.get(type_key)
+ or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type")
+ or "tmt11plex"
+ ).lower()
+
+ m = re.search(r'\d+', selected_type)
+ is_supported_type = any(label in selected_type for label in ["tmt", "itraq"])
+ if not m or not is_supported_type:
+ st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.")
+ else:
+ num_plex = int(m.group())
+ channels = [f"sample{i+1}" for i in range(num_plex)]
+ st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels")
+ st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.")
+
+ for row_start in range(0, num_plex, 2):
+ c1, c2 = st.columns(2)
+
+ left_idx = row_start
+ left_channel = channels[left_idx]
+ with c1:
+ self.ui.input_widget(
+ key=f"TMT-group-{left_channel}",
+ default="",
+ name=f"Group for channel {left_idx + 1}",
+ widget_type="text",
+ help="e.g. control, case, skip",
+ )
+
+ right_idx = row_start + 1
+ if right_idx < num_plex:
+ right_channel = channels[right_idx]
+ with c2:
self.ui.input_widget(
- key=f"TMT-group-{ch}",
+ key=f"TMT-group-{right_channel}",
default="",
- name=f"Group for {ch}",
+ name=f"Group for channel {right_idx + 1}",
widget_type="text",
- help="Enter group name or 'skip' to ignore this channel.",
+ help="e.g. control, case, skip",
)
- # 3. Read values from params.json and construct a dictionary in tmt_group_map format
- # (This can be used later to filter DataFrames in subsequent logic)
- self.params = self.parameter_manager.get_parameters_from_json()
-
- tmt_group_map = {}
- for i, ch in enumerate(all_channels):
- # Retrieve stored value (default is empty string)
- group_val = self.params.get(f"TMT-group-{ch}", "")
- tmt_group_map[str(i)] = group_val
-
- # For data inspection (remove if not needed)
- if st.checkbox("Show current TMT mapping"):
- st.json(tmt_group_map)
-
- # 4. Clean up parameters from unused/previous TMT settings
- all_possible_channel_keys = {f"TMT-group-{ch}" for ch in all_channels}
- orphaned_keys = [
- k for k in self.params.keys()
- if k.startswith("TMT-group-") and k not in all_possible_channel_keys
- ]
-
- if orphaned_keys:
- for key in orphaned_keys:
- del self.params[key]
- self.parameter_manager.save_parameters()
-
- else:
- st.warning("Please select a TMT type in the parameters first.")
+ # Remove orphaned params from a previously selected larger plex
+ self.params = self.parameter_manager.get_parameters_from_json()
+ valid_keys = {f"TMT-group-{ch}" for ch in channels}
+ orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys]
+ if orphaned:
+ for k in orphaned:
+ del self.params[k]
+ self.parameter_manager.save_parameters()
def execution(self) -> bool:
"""
diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py
index 02d7bd4..2d38ad9 100644
--- a/src/common/results_helpers.py
+++ b/src/common/results_helpers.py
@@ -5,11 +5,8 @@
import numpy as np
import streamlit as st
from pathlib import Path
-from scipy.stats import ttest_ind
from pyopenms import IdXMLFile, MSExperiment, MzMLFile
from src.workflow.ParameterManager import ParameterManager
-from statsmodels.stats.multitest import multipletests
-from statsmodels.stats.multitest import multipletests
def get_workflow_dir(workspace):
"""Get the workflow directory path."""
@@ -185,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat
@st.cache_data
-def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
- """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df.
+def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None:
+ """Load CSV and build abundance matrices for downstream preprocessing.
Args:
workspace_path: Path to the workspace directory
csv_mtime: Modification time of CSV file (used as cache key)
+ params_mtime: Modification time of params.json (used as cache key so
+ changing group assignments in Configure invalidates the cache
+ even when the CSV itself hasn't changed)
Returns:
Tuple of (pivot_df, expr_df, group_map) or None if data unavailable
@@ -200,7 +200,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow")
- workflow_params = parameter_manager.get_parameters_from_json()
+ workflow_params = parameter_manager.get_parameters_from_json()
analysis_mode = workflow_params.get("analysis-mode", "LFQ")
if analysis_mode == "LFQ":
@@ -221,7 +221,9 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
if df.empty:
return None
- # Get group mapping from parameters
+ # Get optional group mapping from parameters.
+ # Group information is not required at this stage; statistical testing
+ # happens in the Statistical page.
param_manager = ParameterManager(workflow_dir)
params = param_manager.get_parameters_from_json()
group_map = {
@@ -230,57 +232,23 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
if key.startswith("mzML-group-") and value
}
- if not group_map:
- return None
-
df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False)
- df["Group"] = df["Reference"].map(group_map)
- df = df.dropna(subset=["Group"])
-
- groups = sorted(df["Group"].unique())
-
- if len(groups) < 2:
- return None
-
- group1, group2 = groups[:2]
-
- # Compute statistics per protein
- stats_rows = []
- for protein, protein_df in df.groupby("ProteinName"):
- g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values
- g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values
-
- if len(g1_vals) < 2 or len(g2_vals) < 2:
- pval = np.nan
- else:
- _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False)
-
- mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan
- mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan
-
- log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan
-
- stats_rows.append({
- "ProteinName": protein,
- "log2FC": log2fc,
- "p-value": pval,
- })
-
- stats_df = pd.DataFrame(stats_rows)
- if not stats_df.empty:
- mask = stats_df["p-value"].notna()
- if mask.any():
- _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh")
- stats_df.loc[mask, "p-adj"] = p_adj
- else:
- stats_df["p-adj"] = np.nan
-
- # Order samples by group (group2 first, then group1)
- sample_group_df = df[["Sample", "Group"]].drop_duplicates()
- group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist()
- group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist()
- all_samples = group2_samples + group1_samples
+ # Build sample display order.
+ if group_map:
+ sample_group_df = df[["Sample", "Reference"]].drop_duplicates()
+ sample_group_df["Group"] = sample_group_df["Reference"].map(group_map)
+ grouped_samples = []
+ for grp in sorted(sample_group_df["Group"].dropna().unique()):
+ grouped_samples.extend(
+ sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist()
+ )
+ remaining_samples = [
+ s for s in sorted(df["Sample"].unique()) if s not in grouped_samples
+ ]
+ all_samples = grouped_samples + remaining_samples
+ else:
+ all_samples = sorted(df["Sample"].unique())
# Build pivot table
pivot_list = []
@@ -299,8 +267,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
pivot_list.append(row)
pivot_df = pd.DataFrame(pivot_list)
- pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left")
- pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]]
+ pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]]
# Build expression matrix (log2-transformed)
expr_df = pivot_df.set_index("ProteinName")[all_samples]
@@ -309,7 +276,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
expr_df = expr_df.dropna()
return pivot_df, expr_df, group_map
-
+
else:
if not quant_dir.exists():
return None
@@ -330,7 +297,7 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
# ratio column removal
df = df.loc[:, ~df.columns.str.contains('ratio', case=False)]
-
+
# exclude_indices = st.session_state.get("tmt_exclude_indices", [])
# group_map = st.session_state.get("tmt_group_map", {})
# Get group mapping from parameters
@@ -362,10 +329,6 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
# st.write("exclude_indices:", exclude_indices)
# st.write("group_map:", group_map)
- if not group_map:
- st.warning("โ ๏ธ Group mapping information is missing. Please configure sample groups in the Setup page.")
- return None
-
if exclude_indices:
# st.write("Current columns:", df.columns.tolist())
# st.write("Number of columns:", len(df.columns))
@@ -376,115 +339,25 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None:
else:
df_cleaned = df.copy()
- if group_map:
- # Create new row data (defaulting to empty strings)
- # Create a list with the same length as the column order of df_cleaned
- new_row = [""] * len(df_cleaned.columns)
- new_row[0] = "Group"
-
- # Get the column names of the current dataframe as a list
- current_cols = df_cleaned.columns.tolist()
- original_cols = df.columns.tolist()
-
- for col_name in current_cols[start_column_offset:]:
- # Check the original index position of this column
- original_idx = original_cols.index(col_name) - start_column_offset
- col_pos = current_cols.index(col_name)
- new_row[col_pos] = group_map.get(original_idx, "NA")
-
- # Insert the row at the top of the dataframe
- # Create a new DF and concatenate to prepend the row to existing data
- group_df = pd.DataFrame([new_row], columns=df_cleaned.columns)
- df_with_groups = pd.concat([group_df, df_cleaned], ignore_index=True)
-
- # drop_msg = f"{len(exclude_indices)} channels dropped" if exclude_indices else "No channels dropped"
- # st.success(f"โ
{drop_msg} and Group names have been inserted at the top of the data.")
-
- # st.write("### Data Preview with Group Information")
- # st.dataframe(df_with_groups.head(10))
-
- if group_map and len(set(group_map.values())) >= 2:
- # Prepare data for calculation
- # Extract group information from row 0 of df_with_groups (the newly added Group row)
- # Actual sample data starts from the 5th column (index 4)
- group_info_row = df_with_groups.iloc[0]
-
- # Get unique group names (excluding NA)
- unique_groups = sorted([g for g in set(group_map.values()) if g != "NA"])
- g1_name, g2_name = unique_groups[0], unique_groups[1]
-
- # Extract numerical data for statistical calculation (from row 1 and column index 4 onwards)
- # Convert to numeric type (to prevent calculation errors)
- numeric_data = df_with_groups.iloc[1:, 4:].apply(pd.to_numeric, errors='coerce')
-
- # Column indexing by group
- # Categorize columns based on the values in the Group row
- g1_cols = [col for col in numeric_data.columns if group_info_row[col] == g1_name]
- g2_cols = [col for col in numeric_data.columns if group_info_row[col] == g2_name]
-
- # Calculate log2FC and p-value for each row
- def run_stats(row):
- v1 = row[g1_cols].dropna()
- v2 = row[g2_cols].dropna()
-
- # log2FC (Group2 / Group1)
- m1, m2 = v1.mean(), v2.mean()
- l2fc = np.log2(m2 / m1) if m1 > 0 and m2 > 0 else np.nan
-
- # p-value (T-test)
- if len(v1) > 1 and len(v2) > 1:
- _, pval = ttest_ind(v1, v2, equal_var=False)
- else:
- pval = np.nan
- return pd.Series([l2fc, pval])
-
- stats_results = numeric_data.apply(run_stats, axis=1)
- stats_results.columns = ['log2FC', 'p-value']
- # Add Adjusted p-value (FDR) calculation
- if not stats_results['p-value'].isna().all():
- # Select only rows that contain p-values
- mask = stats_results['p-value'].notna()
- # Apply Benjamini-Hochberg (BH) correction
- _, p_adj, _, _ = multipletests(stats_results.loc[mask, 'p-value'], method='fdr_bh')
- stats_results.loc[mask, 'p-adj'] = p_adj
- else:
- stats_results['p-adj'] = np.nan
-
- # Construct the final dataframe (Based on df_cleaned - excluding the group row)
- # Insert calculation results into the 2nd and 3rd column positions
- pivot_df = df_cleaned.copy()
- pivot_df.insert(1, "log2FC", stats_results['log2FC'].values)
- pivot_df.insert(2, "p-value", stats_results['p-value'].values)
- pivot_df.insert(3, "p-adj", stats_results['p-adj'].values)
-
- # st.success(f"Analysis Complete: {g1_name} (n={len(g1_cols)}) vs {g2_name} (n={len(g2_cols)})")
-
- # Set the first column ('protein') of final_df as the index
- protein_col = pivot_df.columns[0]
- sample_cols = current_cols[start_column_offset:] # Identify actual sample column names
-
- # Select sample columns and create a matrix
- expr_df = pivot_df.set_index(protein_col)[sample_cols]
-
- # Replace 0 with NaN (to prevent log transformation errors)
- expr_df = expr_df.replace(0, np.nan)
-
- # Log2 transformation (data normalization)
- expr_df = np.log2(expr_df + 1)
-
- # Remove proteins (rows) with any missing values
- expr_df = expr_df.dropna()
-
- return pivot_df, expr_df, group_map
- else:
- st.warning("โ ๏ธ At least two distinct groups are required for statistical analysis.")
- else:
- st.warning("โ ๏ธ No group mapping information is set. Please check the Configure page.")
- return None
+ current_cols = df_cleaned.columns.tolist()
+ sample_cols = current_cols[start_column_offset:]
+
+ # Ensure sample columns are numeric for downstream preprocessing/statistics.
+ pivot_df = df_cleaned.copy()
+ if sample_cols:
+ pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce')
+
+ protein_col = pivot_df.columns[0]
+ expr_df = pivot_df.set_index(protein_col)[sample_cols]
+ expr_df = expr_df.replace(0, np.nan)
+ expr_df = np.log2(expr_df + 1)
+ expr_df = expr_df.dropna()
+
+ return pivot_df, expr_df, group_map
def get_abundance_data(workspace: Path) -> tuple | None:
- """Wrapper that handles cache key (workspace + CSV mtime).
+ """Wrapper that handles cache key (workspace + CSV mtime + params mtime).
Args:
workspace: Path to the workspace directory
@@ -503,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None:
return None
csv_mtime = csv_files[0].stat().st_mtime
- return load_abundance_data(str(workspace), csv_mtime)
+
+ params_file = workflow_dir / "params.json"
+ params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0
+
+ return load_abundance_data(str(workspace), csv_mtime, params_mtime)
+
+
+def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str:
+ """Resolve the protein/row identifier column for the active analysis mode.
+
+ LFQ reports always use "ProteinName"; TMT reports use whatever the
+ report's first column is actually named (e.g. "protein").
+ """
+ workflow_dir = get_workflow_dir(workspace)
+ analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ")
+ return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0]
+
+
+def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict:
+ """Normalize group_map into {actual_sample_column_name: group_name}.
+
+ LFQ group_map keys are already clean sample names (optionally with a
+ ".mzML" suffix). TMT group_map keys are 0-based channel indices that must
+ be matched against the report's actual "sampleN[...]" column names.
+ """
+ workflow_dir = get_workflow_dir(workspace)
+ analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ")
+
+ if analysis_mode == "LFQ":
+ return {
+ k[:-5] if k.endswith(".mzML") else k: v
+ for k, v in group_map.items()
+ }
+
+ actual_sample_names = pivot_df.columns.tolist()
+ norm_map = {}
+ for k, v in group_map.items():
+ try:
+ sample_idx = int(k) + 1
+ except (TypeError, ValueError):
+ continue
+ target_substring = f"sample{sample_idx}["
+ real_full_name = next((name for name in actual_sample_names if target_substring in name), None)
+ if real_full_name:
+ norm_map[real_full_name] = v if v and v.strip() else "Unassigned"
+ return norm_map
diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py
index 6a587cd..042c5e1 100644
--- a/src/workflow/CommandExecutor.py
+++ b/src/workflow/CommandExecutor.py
@@ -5,7 +5,7 @@
import threading
from pathlib import Path
from .Logger import Logger
-from .ParameterManager import ParameterManager
+from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini
import sys
import importlib.util
import json
@@ -268,10 +268,13 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool
# Load merged parameters (_defaults + user overrides) for this tool instance
merged_params = self.parameter_manager.get_merged_params(params_key)
+
+ # Load flag parameter names: params.json takes priority (survives session restart),
+ # session_state is the live fallback during the current session.
flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {})
if not flag_map:
flag_map = st.session_state.get("_topp_flag_params", {})
- flag_params = set(flag_map.get(params_key, []))
+ flag_params: set = set(flag_map.get(params_key, []))
# Construct commands for each process
for i in range(n_processes):
@@ -294,30 +297,27 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool
# Add merged TOPP tool parameters (_defaults + user overrides)
for k, v in merged_params.items():
if k in flag_params:
- # CLI flag: include "-k" only when enabled
+ # CLI flag: include "-k" only when truthy, omit when false
if isinstance(v, str):
- is_enabled = v.lower() in {"true", "1", "yes", "on"}
+ is_enabled = v.lower() == "true"
else:
is_enabled = bool(v)
if is_enabled:
command += [f"-{k}"]
continue
- # For non-flag parameters, skip entirely if empty.
- # Note: 0 and 0.0 are valid values, so use explicit checks.
+ # Regular parameter: skip empty/None, append value otherwise
if v == "" or v is None:
continue
command += [f"-{k}"]
if isinstance(v, str) and "\n" in v:
command += v.split("\n")
- elif isinstance(v, bool):
- command += [str(v).lower()]
else:
command += [str(v)]
# Add custom parameters
for k, v in custom_params.items():
if k in flag_params:
if isinstance(v, str):
- is_enabled = v.lower() in {"true", "1", "yes", "on"}
+ is_enabled = v.lower() == "true"
else:
is_enabled = bool(v)
if is_enabled:
@@ -328,20 +328,12 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool
command += [f"-{k}"]
if isinstance(v, list):
command += [str(x) for x in v]
- elif isinstance(v, bool):
- command += [str(v).lower()]
else:
command += [str(v)]
# Add threads parameter for TOPP tools
command += ["-threads", str(threads_per_command)]
commands.append(command)
- for idx, cmd in enumerate(commands):
- # Print list-form command joined into a single string for readability
- print(f" ๐น Command {idx + 1}: {' '.join(cmd)}")
- print("==========================================================\n")
-
-
# Run command(s)
if len(commands) == 1:
return self.run_command(commands[0])
diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py
index 19e8700..2838c1b 100644
--- a/src/workflow/ParameterManager.py
+++ b/src/workflow/ParameterManager.py
@@ -3,8 +3,52 @@
import shutil
import subprocess
import streamlit as st
+import xml.etree.ElementTree as ET
from pathlib import Path
+
+def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]:
+ """
+ Return short parameter paths for every ``- `` in a ParamXML .ini file.
+
+ Paths match the suffix after ``Tool:1:`` in pyOpenMS (e.g. ``algorithm:epd:masstrace_snr_filtering``).
+ """
+ try:
+ root = ET.parse(ini_path).getroot()
+ except (ET.ParseError, OSError):
+ return set()
+
+ def local_tag(el: ET.Element) -> str:
+ t = el.tag
+ return t.rsplit("}", 1)[-1] if isinstance(t, str) and "}" in t else str(t)
+
+ out: set[str] = set()
+
+ def walk(el: ET.Element, parts: tuple[str, ...]) -> None:
+ for ch in el:
+ lt = local_tag(ch)
+ if lt == "NODE":
+ nm = ch.get("name") or ""
+ walk(ch, parts + (nm,))
+ elif lt == "ITEM" and (ch.get("type") or "").lower() == "bool":
+ nm = ch.get("name") or ""
+ segs = [p for p in parts if p]
+ if nm:
+ segs.append(nm)
+ if not segs:
+ continue
+ # Strip tool root NODE name and instance NODE "1" (not part of pyOpenMS short keys)
+ while segs and segs[0] in (tool_stem, "1"):
+ segs.pop(0)
+ if segs:
+ out.add(":".join(segs))
+
+ for ch in root:
+ if local_tag(ch) == "NODE":
+ walk(ch, ())
+ return out
+
+
class ParameterManager:
"""
Manages the parameters for a workflow, including saving parameters to a JSON file,
@@ -29,6 +73,29 @@ def __init__(self, workflow_dir: Path, workflow_name: str = None):
# Store workflow name for preset loading; default to directory stem if not provided
self.workflow_name = workflow_name or workflow_dir.stem
+ def bool_pairs_session_key(self) -> str:
+ """Session state key holding a set of (tool name, param path) for bool TOPP params."""
+ return f"{self.ini_dir.parent.stem}-topp-bool-pairs"
+
+ def get_bool_param_pairs(self) -> set:
+ """Return the cached set of (tool, param path) bool params; empty set if none."""
+ return st.session_state.get(self.bool_pairs_session_key(), set())
+
+ def _merge_bool_params_from_ini(self, tool: str) -> None:
+ """Load tool.ini (XML) and merge type=bool parameter paths into session_state."""
+ ini_path = Path(self.ini_dir, f"{tool}.ini")
+ if not ini_path.exists():
+ return
+ try:
+ sk = self.bool_pairs_session_key()
+ if sk not in st.session_state:
+ st.session_state[sk] = set()
+ for short in bool_param_paths_from_param_xml_ini(ini_path, tool):
+ st.session_state[sk].add((tool, short))
+ except RuntimeError:
+ # No Streamlit session (e.g. plain `python` import)
+ pass
+
def create_ini(self, tool: str) -> bool:
"""
Create an ini file for a TOPP tool if it doesn't exist.
@@ -41,11 +108,14 @@ def create_ini(self, tool: str) -> bool:
"""
ini_path = Path(self.ini_dir, tool + ".ini")
if ini_path.exists():
+ self._merge_bool_params_from_ini(tool)
return True
try:
subprocess.call([tool, "-write_ini", str(ini_path)])
except FileNotFoundError:
return False
+ if ini_path.exists():
+ self._merge_bool_params_from_ini(tool)
return ini_path.exists()
def save_parameters(self) -> None:
@@ -65,7 +135,7 @@ def save_parameters(self) -> None:
# Advanced parameters are only in session state if the view is active
json_params = self.get_parameters_from_json() | json_params
- # get a list of TOPP tools which are in session state
+ # get a list of TOPP tools (or tool instance names) which are in session state
current_topp_tools = list(
set(
[
@@ -75,12 +145,16 @@ def save_parameters(self) -> None:
]
)
)
- # for each TOPP tool, open the ini file
+ # Retrieve the instance-name โ real-tool-name mapping (set by input_TOPP)
+ tool_instance_map = st.session_state.get("_topp_tool_instance_map", {})
+ # for each TOPP tool (or instance name), open the ini file
for tool in current_topp_tools:
- if not self.create_ini(tool):
+ # Resolve instance name to real tool name for create_ini / ini loading
+ real_tool = tool_instance_map.get(tool, tool)
+ if not self.create_ini(real_tool):
# Could not create ini file - skip this tool
continue
- ini_path = Path(self.ini_dir, f"{tool}.ini")
+ ini_path = Path(self.ini_dir, f"{real_tool}.ini")
if tool not in json_params:
json_params[tool] = {}
# load the param object
@@ -92,19 +166,26 @@ def save_parameters(self) -> None:
# Skip display keys used by multiselect widgets
if key.endswith("_display"):
continue
- # get ini_key
- ini_key = key.replace(self.topp_param_prefix, "").encode()
+ # get ini_key โ map instance name back to real tool name
+ ini_key = key.replace(self.topp_param_prefix, "")
+ if tool != real_tool:
+ ini_key = ini_key.replace(f"{tool}:1:", f"{real_tool}:1:", 1)
+ ini_key = ini_key.encode()
# get ini (default) value by ini_key
ini_value = param.getValue(ini_key)
is_list_param = isinstance(ini_value, list)
- # check if value is different from default OR is an empty list parameter
+ # Effective default: _defaults value if present, else ini value
+ short_key = key.split(":1:")[1]
+ defaults = json_params.get("_defaults", {}).get(tool, {})
+ default_value = defaults.get(short_key, ini_value)
+ # check if value is different from effective default OR is an empty list parameter
if (
- (ini_value != value)
- or (key.split(":1:")[1] in json_params[tool])
+ (default_value != value)
+ or (short_key in json_params[tool])
or (is_list_param and not value) # Always save empty list params
):
# store non-default value
- json_params[tool][key.split(":1:")[1]] = value
+ json_params[tool][short_key] = value
# Save to json file
with open(self.params_file, "w", encoding="utf-8") as f:
json.dump(json_params, f, indent=4)
@@ -129,7 +210,7 @@ def get_parameters_from_json(self) -> dict:
except:
st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.")
return {}
-
+
def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict:
"""
Three-layer parameter merge: ini defaults < _defaults < user overrides.
@@ -154,17 +235,20 @@ def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) ->
merged.update(user)
return merged
- def get_topp_parameters(self, tool: str) -> dict:
+ def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict:
"""
Get all parameters for a TOPP tool, merging defaults with user values.
Args:
- tool: Name of the TOPP tool (e.g., "CometAdapter")
+ tool: Name of the TOPP tool executable (e.g., "CometAdapter")
+ tool_instance_name: Optional instance name used for parameter storage
+ (e.g., "IDFilter_step1"). If not provided, defaults to tool name.
Returns:
Dict with parameter names as keys (without tool prefix) and their values.
Returns empty dict if ini file doesn't exist.
"""
+ instance_name = tool_instance_name or tool
ini_path = Path(self.ini_dir, f"{tool}.ini")
if not ini_path.exists():
return {}
@@ -175,18 +259,14 @@ def get_topp_parameters(self, tool: str) -> dict:
# Build dict from ini (extract short key names)
prefix = f"{tool}:1:"
- full_params = {}
+ ini_params = {}
for key in param.keys():
key_str = key.decode() if isinstance(key, bytes) else str(key)
if prefix in key_str:
short_key = key_str.split(prefix, 1)[1]
- full_params[short_key] = param.getValue(key)
-
- # Override with user-modified values from JSON
- user_params = self.get_parameters_from_json().get(tool, {})
- full_params.update(user_params)
+ ini_params[short_key] = param.getValue(key)
- return full_params
+ return self.get_merged_params(instance_name, ini_params=ini_params)
def reset_to_default_parameters(self) -> None:
"""
diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py
index befde2e..77ca01d 100644
--- a/src/workflow/StreamlitUI.py
+++ b/src/workflow/StreamlitUI.py
@@ -23,6 +23,32 @@
from src.workflow._log_status import classify_log_outcome
+def _mounted_data_root() -> Union[Path, None]:
+ """Return the validated mount root from the ``local_data_dir`` setting.
+
+ The browser renders only when ``local_data_dir`` is an actual mount
+ point inside the container โ i.e. the operator passed ``-v`` /
+ ``--bind`` / ``volumeMount`` to attach host data. Existence alone is
+ no longer sufficient because the image now pre-creates the path so
+ apptainer/singularity binds have a real attach target; without
+ ``os.path.ismount`` the browser would render an empty tree for every
+ user who didn't mount anything.
+ """
+ settings = st.session_state.get("settings") or {}
+ raw = (settings.get("local_data_dir") or "").strip()
+ if not raw:
+ return None
+ try:
+ p = Path(raw).expanduser().resolve(strict=True)
+ except (OSError, RuntimeError):
+ return None
+ if not p.is_dir():
+ return None
+ if not os.path.ismount(p):
+ return None
+ return p
+
+
class StreamlitUI:
"""
Provides an interface for Streamlit applications to handle file uploads,
@@ -77,6 +103,8 @@ def upload_widget(
c1, c2 = st.columns(2)
c1.markdown("**Upload file(s)**")
+ mount_root = _mounted_data_root() if st.session_state.location == "online" else None
+
if st.session_state.location == "local":
c2_text, c2_checkbox = c2.columns([1.5, 1], gap="large")
c2_text.markdown("**OR add files from local folder**")
@@ -247,7 +275,19 @@ def upload_widget(
"This means that the original files will be used instead. "
)
- if fallback and not any([f for f in Path(files_dir).iterdir() if f.name != "external_files.txt"]):
+ if mount_root is not None:
+ with c2:
+ self._mounted_drive_browser(key, name, file_types, files_dir, mount_root)
+
+ external_files_path = Path(files_dir, "external_files.txt")
+ has_real_files = any(
+ p.name != "external_files.txt" for p in files_dir.iterdir()
+ )
+ has_external_picks = external_files_path.exists() and any(
+ line.strip() and os.path.exists(line.strip())
+ for line in external_files_path.read_text().splitlines()
+ )
+ if fallback and not has_real_files and not has_external_picks:
if isinstance(fallback, str):
fallback = [fallback]
for f in fallback:
@@ -303,6 +343,179 @@ def upload_widget(
elif not fallback:
st.warning(f"No **{name}** files!")
+ def _resolve_browser_cwd(self, key: str, mount_root: Path) -> Path:
+ """Read cwd for this widget from session state, confine it to mount_root."""
+ sess_key = f"mounted_cwd_{key}"
+ raw = st.session_state.get(sess_key, str(mount_root))
+ try:
+ cwd = Path(raw).expanduser().resolve(strict=True)
+ except (OSError, RuntimeError):
+ cwd = mount_root
+ if cwd != mount_root and mount_root not in cwd.parents:
+ cwd = mount_root
+ st.session_state[sess_key] = str(cwd)
+ return cwd
+
+ def _mounted_drive_browser(
+ self,
+ key: str,
+ name: str,
+ file_types: List[str],
+ files_dir: Path,
+ mount_root: Path,
+ ) -> None:
+ """Render a tree browser for a mounted host directory.
+
+ Selected files are referenced in place via ``external_files.txt`` โ
+ the same mechanism the offline tkinter flow uses.
+ """
+ external_files = Path(files_dir, "external_files.txt")
+ if not external_files.exists():
+ external_files.touch()
+
+ cwd = self._resolve_browser_cwd(key, mount_root)
+ sess_cwd_key = f"mounted_cwd_{key}"
+
+ st.markdown(
+ """
+
+ """,
+ unsafe_allow_html=True,
+ )
+
+ with st.container(border=True):
+ st.markdown(
+ f"**Add {name} files from mounted directory** "
+ f"`{mount_root}`"
+ )
+
+ # Breadcrumbs: compact tertiary buttons separated by ยป,
+ # with a right-aligned Parent button.
+ try:
+ rel = cwd.relative_to(mount_root)
+ segments = [mount_root.name] + list(rel.parts) if rel.parts else [mount_root.name]
+ except ValueError:
+ segments = [mount_root.name]
+ n = len(segments)
+ ratios: List[float] = []
+ for i in range(n):
+ ratios.append(max(len(segments[i]), 3))
+ if i < n - 1:
+ ratios.append(1)
+ ratios.append(20) # flexible spacer
+ ratios.append(6) # parent button slot
+ crumb_cols = st.columns(ratios, vertical_alignment="center")
+ col_idx = 0
+ for i, seg in enumerate(segments):
+ target = mount_root.joinpath(*segments[1 : i + 1]) if i > 0 else mount_root
+ if crumb_cols[col_idx].button(
+ seg,
+ key=f"crumb_{key}_{i}",
+ type="tertiary",
+ ):
+ st.session_state[sess_cwd_key] = str(target)
+ st.rerun(scope="fragment")
+ col_idx += 1
+ if i < n - 1:
+ crumb_cols[col_idx].markdown(
+ "ยป",
+ unsafe_allow_html=True,
+ )
+ col_idx += 1
+ # spacer column
+ col_idx += 1
+ if cwd != mount_root:
+ if crumb_cols[col_idx].button(
+ "โฌ Parent",
+ key=f"mounted_parent_{key}",
+ type="tertiary",
+ ):
+ st.session_state[sess_cwd_key] = str(cwd.parent)
+ st.rerun(scope="fragment")
+
+ try:
+ entries = sorted(
+ (p for p in cwd.iterdir() if not p.name.startswith(".")),
+ key=lambda p: (not p.is_dir(), p.name.lower()),
+ )
+ except PermissionError:
+ st.error(f"Permission denied reading `{cwd}`.")
+ return
+
+ def _is_match(p: Path) -> bool:
+ return any(p.name.endswith(f".{ft}") for ft in file_types)
+
+ subdirs = [p for p in entries if p.is_dir() and not _is_match(p)]
+ bundled = [p for p in entries if p.is_dir() and _is_match(p)]
+ files = [p for p in entries if p.is_file() and _is_match(p)]
+
+ for d in subdirs:
+ indent, body = st.columns([1, 60], vertical_alignment="center")
+ if body.button(
+ f"๐ {d.name}/",
+ key=f"mounted_dir_{key}_{d.name}",
+ type="tertiary",
+ ):
+ st.session_state[sess_cwd_key] = str(d)
+ st.rerun(scope="fragment")
+
+ selectable = bundled + files
+ selected_paths: List[str] = []
+ for f in selectable:
+ cb_key = f"mounted_pick_{key}_{f}"
+ size_label = ""
+ if f.is_file():
+ try:
+ size_mb = f.stat().st_size / (1024 * 1024)
+ size_label = f" ยท {size_mb:.1f} MB"
+ except OSError:
+ pass
+ icon = "๐๏ธ" if f.is_dir() else "๐"
+ if st.checkbox(
+ f"{icon} {f.name}{size_label}",
+ key=cb_key,
+ ):
+ selected_paths.append(str(f))
+
+ if not subdirs and not selectable:
+ st.info(
+ f"No subdirectories or files matching "
+ f"**{', '.join('.' + ft for ft in file_types)}** here."
+ )
+
+ count = len(selected_paths)
+ if st.button(
+ f"โ Add {count} selected {name} file(s)" if count else f"โ Add selected {name} file(s)",
+ key=f"mounted_add_{key}",
+ type="primary",
+ use_container_width=True,
+ disabled=count == 0,
+ ):
+ existing = set(
+ line.strip()
+ for line in external_files.read_text().splitlines()
+ if line.strip()
+ )
+ added = 0
+ with open(external_files, "a") as fh:
+ for p in selected_paths:
+ if p not in existing:
+ fh.write(f"{p}\n")
+ existing.add(p)
+ added += 1
+ # Clear the checkboxes by removing their session keys.
+ for f in selectable:
+ st.session_state.pop(f"mounted_pick_{key}_{f}", None)
+ st.success(f"Added {added} file(s) from `{cwd}`.")
+ st.rerun(scope="fragment")
+
def select_input_file(
self,
key: str,
@@ -606,7 +819,6 @@ def format_files(input: Any) -> List[str]:
self.parameter_manager.save_parameters()
- @st.fragment
def input_TOPP(
self,
topp_tool_name: str,
@@ -619,6 +831,79 @@ def input_TOPP(
display_subsection_tabs: bool = False,
custom_defaults: dict = {},
tool_instance_name: str = None,
+ reactive: bool = False,
+ ) -> None:
+ """
+ Wrapper for TOPP parameter input. When `reactive` is True the
+ implementation is rendered directly in the parent context so changes
+ trigger a parent re-render; otherwise the widgets are rendered inside
+ a `st.fragment` to isolate reruns for performance.
+ """
+ if reactive:
+ return self._input_TOPP_impl(
+ topp_tool_name=topp_tool_name,
+ num_cols=num_cols,
+ exclude_parameters=exclude_parameters,
+ include_parameters=include_parameters,
+ flag_parameters=flag_parameters,
+ display_tool_name=display_tool_name,
+ display_subsections=display_subsections,
+ display_subsection_tabs=display_subsection_tabs,
+ custom_defaults=custom_defaults,
+ tool_instance_name=tool_instance_name,
+ )
+ return self._input_TOPP_fragmented(
+ topp_tool_name=topp_tool_name,
+ num_cols=num_cols,
+ exclude_parameters=exclude_parameters,
+ include_parameters=include_parameters,
+ flag_parameters=flag_parameters,
+ display_tool_name=display_tool_name,
+ display_subsections=display_subsections,
+ display_subsection_tabs=display_subsection_tabs,
+ custom_defaults=custom_defaults,
+ tool_instance_name=tool_instance_name,
+ )
+
+ @st.fragment
+ def _input_TOPP_fragmented(
+ self,
+ topp_tool_name: str,
+ num_cols: int = 4,
+ exclude_parameters: List[str] = [],
+ include_parameters: List[str] = [],
+ flag_parameters: List[str] = [],
+ display_tool_name: bool = True,
+ display_subsections: bool = True,
+ display_subsection_tabs: bool = False,
+ custom_defaults: dict = {},
+ tool_instance_name: str = None,
+ ) -> None:
+ return self._input_TOPP_impl(
+ topp_tool_name=topp_tool_name,
+ num_cols=num_cols,
+ exclude_parameters=exclude_parameters,
+ include_parameters=include_parameters,
+ flag_parameters=flag_parameters,
+ display_tool_name=display_tool_name,
+ display_subsections=display_subsections,
+ display_subsection_tabs=display_subsection_tabs,
+ custom_defaults=custom_defaults,
+ tool_instance_name=tool_instance_name,
+ )
+
+ def _input_TOPP_impl(
+ self,
+ topp_tool_name: str,
+ num_cols: int = 4,
+ exclude_parameters: List[str] = [],
+ include_parameters: List[str] = [],
+ flag_parameters: List[str] = [],
+ display_tool_name: bool = True,
+ display_subsections: bool = True,
+ display_subsection_tabs: bool = False,
+ custom_defaults: dict = {},
+ tool_instance_name: str = None,
) -> None:
"""
Generates input widgets for TOPP tool parameters dynamically based on the tool's
@@ -630,8 +915,6 @@ def input_TOPP(
num_cols (int, optional): Number of columns to use for the layout. Defaults to 3.
exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list.
include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list.
- flag_parameters (List[str], optional): List of parameter names that should
- be treated as no-value CLI flags during command construction.
display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True.
display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True.
display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False.
@@ -651,16 +934,18 @@ def input_TOPP(
if "_topp_tool_instance_map" not in st.session_state:
st.session_state["_topp_tool_instance_map"] = {}
st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name
+
+ # Persist flag_parameters to session_state and params.json so run_topp
+ # can skip appending a value for these boolean CLI flags.
if "_topp_flag_params" not in st.session_state:
st.session_state["_topp_flag_params"] = {}
st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters)
- # Persist flag metadata so execution still sees it outside UI reruns/session context.
- params = self.parameter_manager.get_parameters_from_json()
- if "_flag_params" not in params:
- params["_flag_params"] = {}
- params["_flag_params"][tool_instance_name] = list(flag_parameters)
- with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f:
- json.dump(params, f, indent=4)
+ _fp = self.parameter_manager.get_parameters_from_json()
+ if "_flag_params" not in _fp:
+ _fp["_flag_params"] = {}
+ _fp["_flag_params"][tool_instance_name] = list(flag_parameters)
+ with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f:
+ json.dump(_fp, _f, indent=4)
if not display_subsections:
display_subsection_tabs = False
@@ -752,7 +1037,6 @@ def _matches_parameter(pattern: str, key: bytes) -> bool:
":".join(key.decode().split(":")[:-1])
),
}
- p["is_flag"] = (b"flag" in param.getTags(key))
# Parameter sections and subsections as string (e.g. "section:subsection")
if display_subsections:
p["sections"] = ":".join(
@@ -843,43 +1127,16 @@ def display_TOPP_params(params: dict, num_cols):
# sometimes strings with newline, handle as list
if isinstance(p["value"], str) and "\n" in p["value"]:
p["value"] = p["value"].split("\n")
- # no-value CLI flag parameters should be shown as checkboxes
- if p.get("is_flag", False):
- flag_default = p["value"]
- if isinstance(flag_default, str):
- flag_default = flag_default.lower() in {"true", "1", "yes", "on"}
- else:
- flag_default = bool(flag_default)
- # Streamlit widget keys persist in session_state and can override
- # updated custom_defaults. Normalize and seed key explicitly.
- if key in st.session_state:
- current = st.session_state[key]
- if isinstance(current, str):
- st.session_state[key] = current.lower() in {"true", "1", "yes", "on"}
- else:
- st.session_state[key] = bool(current)
- else:
- st.session_state[key] = flag_default
- cols[i].selectbox(
- name,
- options=[True, False],
- index=0 if st.session_state[key] else 1,
- format_func=lambda x: "True" if x else "False",
- help=p["description"],
- key=key,
- )
# bools
- elif isinstance(p["value"], bool):
- bool_value = (
- (p["value"] == "true")
- if type(p["value"]) == str
- else p["value"]
- )
- cols[i].selectbox(
+ if isinstance(p["value"], bool):
+ cols[i].markdown("##")
+ cols[i].checkbox(
name,
- options=[True, False],
- index=0 if bool_value else 1,
- format_func=lambda x: "True" if x else "False",
+ value=(
+ (p["value"] == "true")
+ if type(p["value"]) == str
+ else p["value"]
+ ),
help=p["description"],
key=key,
)
@@ -964,6 +1221,7 @@ def on_multiselect_change(dk=display_key, tk=key):
cols[i].error(f"Error in parameter **{p['name']}**.")
print('Error parsing "' + p["name"] + '": ' + str(e))
+
for section, params in param_sections.items():
if tabs is None:
show_subsection_header(section, display_subsections)
@@ -1435,7 +1693,8 @@ def remove_full_paths(d: dict) -> dict:
general = {}
for k, v in params.items():
- # skip if v is a file path
+ if k == "_defaults":
+ continue
if isinstance(v, dict):
topp[k] = v
elif ".py" in k:
@@ -1446,6 +1705,13 @@ def remove_full_paths(d: dict) -> dict:
else:
general[k] = v
+ # Merge _defaults into topp so summary shows custom defaults + user overrides
+ defaults = params.get("_defaults", {})
+ for tool_name, default_vals in defaults.items():
+ if tool_name not in topp:
+ topp[tool_name] = {}
+ topp[tool_name] = {**default_vals, **topp.get(tool_name, {})}
+
markdown = []
def dict_to_markdown(d: dict):