From 3016b0a2c24599da1a9a5e9cde827075d0431a30 Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 10:34:59 +0900 Subject: [PATCH 1/7] Revert "feat: integrate LFQ and TMT workflows (#27)" This reverts commit 83646ecb68387b2da2f0bb57c8f6aa1c693dbe15. --- app.py | 2 +- content/results_abundance.py | 135 +- content/results_heatmap.py | 130 +- content/results_pathway_analysis.py | 258 ---- content/results_pca.py | 45 +- content/results_volcano.py | 240 ++-- src/WorkflowTest.py | 1829 +++++++++------------------ src/common/results_helpers.py | 351 ++--- src/workflow/CommandExecutor.py | 81 +- src/workflow/ParameterManager.py | 24 - src/workflow/StreamlitUI.py | 126 +- 11 files changed, 915 insertions(+), 2306 deletions(-) delete mode 100644 content/results_pathway_analysis.py diff --git a/app.py b/app.py index 97f9a16..194d857 100644 --- a/app.py +++ b/app.py @@ -27,7 +27,7 @@ st.Page(Path("content", "results_pca.py"), title="PCA", icon="πŸ“Š"), st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="πŸ”₯"), st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="πŸ“š"), - st.Page(Path("content", "results_pathway_analysis.py"), title="Pathway Analysis", icon="πŸ“‰"), + st.Page(Path("content", "results_proteomicslfq.py"), title="Proteomics LFQ", icon="πŸ§ͺ"), ], } diff --git a/content/results_abundance.py b/content/results_abundance.py index a86f1a3..a7ff453 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -4,7 +4,6 @@ from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data -from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -22,10 +21,6 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -40,60 +35,6 @@ csv_file = csv_files[0] -def render_protein_table(pivot_df, group_map, is_lfq=True): - """Common function to render the protein-level abundance table""" - st.markdown("### Protein-Level Abundance Table") - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) - - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") - - if is_lfq: - # Handle LFQ mode columns (Raw Intensity) - id_col = "ProteinName" - exclude_cols = [id_col, "log2FC", "p-value", "PeptideSequence"] - sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] - - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - help_text = "Raw sample intensities" - y_min = None - else: - # Handle non-LFQ mode columns (Log2-transformed Intensity) - id_col = "protein" - exclude_cols = [id_col, "log2FC", "p-value", "p-adj", "n_proteins", "n_peptides", "protein_score"] - sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] - - pivot_df["Intensity"] = pivot_df[sample_cols].apply( - lambda row: [np.log2(v + 1) for v in row], axis=1 - ) - display_cols = [id_col, "log2FC", "p-value", "Intensity"] + sample_cols - help_text = "Sample intensities (log2 scale)" - y_min = 0 - - # Filter to available columns, then sort and display - available_cols = [c for c in display_cols if c in pivot_df.columns] - - st.dataframe( - pivot_df[available_cols].sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help=help_text, - width="small", - y_min=y_min, - ), - }, - use_container_width=True, - ) - protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -103,44 +44,58 @@ def render_protein_table(pivot_df, group_map, is_lfq=True): st.info("No data found in this file.") st.stop() - result = get_abundance_data(st.session_state["workspace"]) + with protein_tab: + st.markdown("### Protein-Level Abundance Table") - if analysis_mode == "LFQ": - protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - - with protein_tab: - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - # st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() - - pivot_df, expr_df, group_map = result - render_protein_table(pivot_df, group_map, is_lfq=True) - - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) - - else: - pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + result = get_abundance_data(st.session_state["workspace"]) if result is None: - st.info("πŸ’‘ Please complete the configuration in the 'Configure' page to see results.") + st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") + st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") st.stop() - + pivot_df, expr_df, group_map = result - with pre_processing_tab: - st.write("### Final Results (Group row removed, Stats added)") - st.dataframe(pivot_df.head(10)) + # Display group comparison info + groups = sorted(set(group_map.values())) + if len(groups) >= 2: + group1, group2 = sorted(groups)[:2] + st.info(f"Statistical comparison: **{group2} vs {group1}**") + + # Get sample columns (between stats and PeptideSequence) + sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) - with protein_tab: - render_protein_table(pivot_df, group_map, is_lfq=False) + # Reorder columns: place Intensity after p-value + display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] + display_df = pivot_df[display_cols] + + st.dataframe( + display_df.sort_values("p-value"), + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help="Raw sample intensities", + width="small", + ), + }, + use_container_width=True, + ) + + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 72b8438..4ece3f4 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -5,8 +5,7 @@ from scipy.cluster.hierarchy import linkage, leaves_list from scipy.spatial.distance import pdist from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("Heatmap") @@ -30,103 +29,48 @@ pivot_df, expr_df, group_map = result -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") +top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() -st.write("Workflow Analysis Mode:", analysis_mode) +if not heatmap_z.empty: + row_linkage = linkage(pdist(heatmap_z.values), method="average") + row_order = leaves_list(row_linkage) -if analysis_mode == "LFQ": - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") + col_linkage = linkage(pdist(heatmap_z.T.values), method="average") + col_order = leaves_list(col_linkage) - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + heatmap_clustered = heatmap_z.iloc[row_order, col_order] - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + fig_heatmap = px.imshow( + heatmap_clustered, + labels=dict(x="Sample", y="Protein", color="Z-score"), + aspect="auto", + color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], + zmin=-3, zmax=3 + ) - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + fig_heatmap.update_layout( + height=700, + xaxis={'side': 'bottom'}, + yaxis={'side': 'left'} + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + fig_heatmap.update_xaxes(tickfont=dict(size=10)) + fig_heatmap.update_yaxes(tickfont=dict(size=8)) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) - else: - st.warning("Insufficient data to generate the heatmap.") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.plotly_chart(fig_heatmap, use_container_width=True) else: - top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") - - var_series = expr_df.var(axis=1) - top_proteins = var_series.sort_values(ascending=False).head(top_n).index - heatmap_df = expr_df.loc[top_proteins] - heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) - heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() - - if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) - - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) - - heatmap_clustered = heatmap_z.iloc[row_order, col_order] - - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) - - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} - ) - - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, width="stretch") - else: - st.warning("Insufficient data to generate the heatmap.") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") - with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.warning("Insufficient data to generate the heatmap.") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py deleted file mode 100644 index f5eb4c1..0000000 --- a/content/results_pathway_analysis.py +++ /dev/null @@ -1,258 +0,0 @@ -import json -import mygene -import streamlit as st -import pandas as pd -import numpy as np -import plotly.express as px -import plotly.io as pio -from collections import defaultdict -from scipy.stats import fisher_exact -from pathlib import Path -from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data - -# ================================ -# Page setup -# ================================ -params = page_setup() -st.title("ProteomicsLFQ Results") - -# ================================ -# Workspace check -# ================================ -if "workspace" not in st.session_state: - st.warning("Please initialize your workspace first.") - st.stop() - -# ================================ -# _run_go_enrichment function -# ================================ -def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - st.write("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - # st.write("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - st.write("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) - - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - # st.write(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) - - # st.write(f"βœ… Plotly Figure generated") - - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) - - return fig, df - - go_results = {} - - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - # st.write(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - # st.write("βœ… GO enrichment analysis complete") - -# ================================ -# Load abundance data -# ================================ -results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" -result = get_abundance_data(st.session_state["workspace"]) -if result is None: - st.info("Abundance data not available. Please run the workflow and configure sample groups first.") - st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") - st.stop() - -pivot_df, expr_df, group_map = result - -go_json_file = results_dir / "go-terms" / "go_results.json" - -go_input_df = pivot_df.copy() -if "ProteinName" in go_input_df.columns: - go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) - -_run_go_enrichment(go_input_df, results_dir) - -# ================================ -# Tabs -# ================================ -protein_tab, = st.tabs(["🧬 Protein Table"]) - -# ================================ -# Protein-level results -# ================================ -with protein_tab: - st.markdown("### 🧬 Protein-Level Abundance Table") - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) - - if pivot_df.empty: - st.info("No protein-level data available.") - else: - st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), width="stretch") - -# ====================================================== -# GO Enrichment Results -# ====================================================== -st.markdown("---") -st.subheader("🧬 GO Enrichment Analysis") - -if not go_json_file.exists(): - st.info("GO Enrichment results are not available yet. Please run the analysis first.") -else: - with open(go_json_file, "r") as f: - go_data = json.load(f) - - bp_tab, cc_tab, mf_tab = st.tabs([ - "🧬 Biological Process", - "🏠 Cellular Component", - "βš™οΈ Molecular Function", - ]) - - for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): - with tab: - if go_type not in go_data: - st.info(f"No enriched {go_type} terms found.") - continue - - fig_json = go_data[go_type]["fig_json"] - df_dict = go_data[go_type]["df_dict"] - - fig = pio.from_json(fig_json) - - df_go = pd.DataFrame(df_dict) - - if df_go.empty: - st.info(f"No enriched {go_type} terms found.") - else: - st.plotly_chart(fig, width="stretch") - - st.markdown(f"#### {go_type} Enrichment Results") - st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 466e475..45ea8eb 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -5,8 +5,7 @@ from sklearn.decomposition import PCA from sklearn.preprocessing import StandardScaler from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("PCA Analysis") @@ -30,25 +29,13 @@ pivot_df, expr_df, group_map = result -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - top_n = 500 -if analysis_mode == "LFQ": - protein_col = "ProteinName" -else: - protein_col = "protein" - top_proteins = ( pivot_df .dropna(subset=["p-adj"]) .sort_values("p-adj", ascending=True) - .head(top_n)[protein_col] + .head(top_n)["ProteinName"] ) expr_df_pca = expr_df.loc[ @@ -71,25 +58,10 @@ index=X.index ) -if analysis_mode == "LFQ": - norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() - } -else: - actual_sample_names = pca_df.index.tolist() - norm_map = {} - for k, v in group_map.items(): - try: - sample_idx = int(k) + 1 - target_substring = f"sample{sample_idx}[" - real_full_name = next((name for name in actual_sample_names if target_substring in name), None) - - if real_full_name: - norm_map[real_full_name] = v if v and v.strip() else "Unassigned" - except ValueError: - continue - +norm_map = { + k.replace(".mzML", ""): v + for k, v in group_map.items() +} pca_df["Group"] = pca_df.index.map(norm_map) fig_pca = px.scatter( @@ -107,9 +79,8 @@ height=600, ) -st.plotly_chart(fig_pca, width="stretch") +st.plotly_chart(fig_pca, use_container_width=True) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") st.markdown("---") @@ -118,4 +89,4 @@ with col1: st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") \ No newline at end of file + st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") diff --git a/content/results_volcano.py b/content/results_volcano.py index 5cc9b29..8502489 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -3,8 +3,7 @@ import plotly.express as px import numpy as np from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data, get_workflow_dir -from src.workflow.ParameterManager import ParameterManager +from src.common.results_helpers import get_abundance_data params = page_setup() st.title("Volcano Plot") @@ -29,164 +28,79 @@ pivot_df, expr_df, group_map = result if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -workflow_dir = get_workflow_dir(st.session_state["workspace"]) -parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - -workflow_params = parameter_manager.get_parameters_from_json() -analysis_mode = workflow_params.get("analysis-mode", "LFQ") - -st.write("Workflow Analysis Mode:", analysis_mode) - -if analysis_mode == "LFQ": - volcano_df = pivot_df.copy() - volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.5, - max_value=3.0, - value=1.0, - step=0.1, - ) - - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, use_container_width=True) - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") -else: - # Threshold Selection UI - st.divider() - c1, c2 = st.columns(2) - with c1: - fc_thresh = st.slider( - "log2 Fold Change threshold", - min_value=0.1, - max_value=3.0, - value=1.0, - step=0.1, - ) - with c2: - p_thresh = st.slider( - "p-adj (FDR) threshold", - min_value=0.001, - max_value=0.1, - value=0.05, - step=0.001, - ) - - volcano_df = pivot_df.dropna(subset=["log2FC", "p-adj"]).copy() - volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) - - volcano_df["Significance"] = "Not significant" - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", - ] = "Up-regulated" - - volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", - ] = "Down-regulated" - - fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["protein", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } - ) - - fig_volcano.add_vline(x=fc_thresh, line_dash="dash") - fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") - fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - - # Make x-axis symmetric around zero - max_abs_fc = volcano_df["log2FC"].abs().max() - x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding - - fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, - height=600, - ) - - st.plotly_chart(fig_volcano, width="stretch") - - up_count = (volcano_df["Significance"] == "Up-regulated").sum() - down_count = (volcano_df["Significance"] == "Down-regulated").sum() - st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") - - st.markdown("---") - st.markdown("**Other visualizations:**") - col1, col2 = st.columns(2) - with col1: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") - with col2: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.info("No data available for volcano plot.") + st.stop() + +volcano_df = pivot_df.copy() +volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) + +volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) + +fc_thresh = st.slider( + "log2 Fold Change threshold", + min_value=0.5, + max_value=3.0, + value=1.0, + step=0.1, +) + +p_thresh = st.slider( + "p-adj (FDR) threshold", + min_value=0.001, + max_value=0.1, + value=0.05, + step=0.001, +) + +volcano_df["Significance"] = "Not significant" +volcano_df.loc[ + (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), + "Significance", +] = "Up-regulated" + +volcano_df.loc[ + (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), + "Significance", +] = "Down-regulated" + +fig_volcano = px.scatter( + volcano_df, + x="log2FC", + y="neg_log10_padj", + color="Significance", + hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], + color_discrete_map={ + "Up-regulated": "red", + "Down-regulated": "blue", + "Not significant": "lightgrey", + } +) + +fig_volcano.add_vline(x=fc_thresh, line_dash="dash") +fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") +fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") + +# Make x-axis symmetric around zero +max_abs_fc = volcano_df["log2FC"].abs().max() +x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding + +fig_volcano.update_layout( + xaxis_title="log2 Fold Change", + yaxis_title="-log10(p-adj)", + xaxis_range=x_range, + height=600, +) + +st.plotly_chart(fig_volcano, use_container_width=True) + +up_count = (volcano_df["Significance"] == "Up-regulated").sum() +down_count = (volcano_df["Significance"] == "Down-regulated").sum() +st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index ebf7e65..4abbf92 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -47,29 +47,6 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) - self.params = self.parameter_manager.get_parameters_from_json() - saved_mode = self.params.get("analysis-mode", "LFQ") - - self.ui.input_widget( - key="analysis-mode", - default=saved_mode, - name="Analysis Mode", - widget_type="selectbox", - options=["LFQ", "TMT"], - help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", - reactive=True - ) - - self.params = self.parameter_manager.get_parameters_from_json() - current_mode = self.params.get("analysis-mode", "LFQ") - - if current_mode == "LFQ": - self.render_lfq_tabs() - else: - self.render_tmt_tabs() - - def render_lfq_tabs(self): - st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -93,10 +70,10 @@ def render_lfq_tabs(self): st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -125,7 +102,7 @@ def render_lfq_tabs(self): st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -134,7 +111,7 @@ def render_lfq_tabs(self): "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "low_res", + "instrument": "high_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -143,19 +120,16 @@ def render_lfq_tabs(self): "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.6, - "fragment_bin_offset": 0.4, + "fragment_mass_tolerance": 0.02, + "fragment_bin_offset": 0.0, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", - "PeptideIndexing:IL_equivalent": True, + "PeptideIndexing:IL_equivalent": "true", "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", - "mass_recalibration": False, }, include_parameters=comet_include, - flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -176,10 +150,9 @@ def render_lfq_tabs(self): "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": True, + "post_processing_tdc": "true", }, include_parameters=percolator_include, - flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -275,10 +248,6 @@ def render_lfq_tabs(self): "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", - "alignment_order": "star", - "protein_quantification": "unique_peptides", - "quantification_method": "feature_intensity", - "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -329,295 +298,6 @@ def render_lfq_tabs(self): if orphaned_keys: self.parameter_manager.save_parameters() - def render_tmt_tabs(self): - st.subheader("TMT Analysis Mode") - # Create tabs for different analysis steps. - t = st.tabs( - ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", - "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] - ) - with t[0]: - # Checkbox for decoy generation - # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, - # so conditional UI (DecoyDatabase settings) updates immediately - self.ui.input_widget( - key="generate-decoys", - default=True, - name="Generate Decoy Database", - widget_type="checkbox", - help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", - reactive=True, - ) - - # Reload params to get current checkbox value after it was saved - self.params = self.parameter_manager.get_parameters_from_json() - - # Show DecoyDatabase settings if generating decoys - if self.params.get("generate-decoys", True): - st.info(""" - **Decoy Database Settings:** - * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. - """) - self.ui.input_TOPP( - "DecoyDatabase", - custom_defaults={ - "decoy_string": "rev_", - "decoy_string_position": "prefix", - "method": "reverse", - }, - include_parameters=["method"], - ) - - comet_info = """ - **Identification (Comet):** - * **enzyme**: The enzyme used for peptide digestion. - * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. - * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' - * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' - * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. - * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. - * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. - """ - if not self.params.get("generate-decoys", True): - comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions - in the protein database to indicate decoy proteins. - """ - st.info(comet_info) - - st.write(Path(self.workflow_dir, "results")) - - comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] - if not self.params.get("generate-decoys", True): - # Only show decoy_string when not generating decoys - comet_include.append("PeptideIndexing:decoy_string") - - self.ui.input_TOPP( - "IsobaricAnalyzer", - custom_defaults={ - "tmt11plex:reference_channel": 126, - "type": "tmt11plex", - "extraction:select_activation": "auto", - "extraction:reporter_mass_shift": 0.002, - "extraction:min_reporter_intensity": 0.0, - "extraction:min_precursor_purity": 0.0, - "extraction:precursor_isotope_deviation": 10.0, - "quantification:isotope_correction": "false", - }, - tool_instance_name="IsobaricAnalyzer-TMT", - ) - with t[1]: - comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] - self.ui.input_TOPP( - "CometAdapter", - custom_defaults={ - "PeptideIndexing:IL_equivalent": True, - "clip_nterm_methionine": "true", - "instrument": "high_res", - "missed_cleavages": 2, - "min_peptide_length": 6, - "max_peptide_length": 40, - "enzyme": "Trypsin/P", - "PeptideIndexing:unmatched_action": "warn", - "max_variable_mods_in_peptide": 3, - "precursor_mass_tolerance": 4.5, - "isotope_error": "0/1", - "precursor_error_units": "ppm", - "num_hits": 1, - "num_enzyme_termini": "fully", - "fragment_bin_offset": 0.0, - "minimum_peaks": 10, - "precursor_charge": "2:4", - "fragment_mass_tolerance": 0.015, - "PeptideIndexing:unmatched_action": "warn", - "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", - "debug": 0, - "force": True, - }, - include_parameters=comet_include, - flag_parameters=["PeptideIndexing:IL_equivalent", "force"], - exclude_parameters=["second_enzyme"], - tool_instance_name="CometAdapter-TMT", - ) - with t[2]: - st.info(""" - **Filtering (IDFilter):** - * **score:type_peptide**: Score used for filtering. If empty, the main score is used. - * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) - """) - self.ui.input_TOPP( - "PercolatorAdapter", - custom_defaults={ - "subset_max_train": 300000, - "decoy_pattern": "DECOY_", - "score_type": "pep", - "post_processing_tdc": True, - "debug": 0, - }, - flag_parameters=["post_processing_tdc"], - tool_instance_name="PercolatorAdapter-TMT", - ) - - with t[3]: - self.ui.input_TOPP( - "IDFilter", - custom_defaults={ - "score:type_peptide": "q-value", - "score:psm": 0.10, - }, - tool_instance_name="IDFilter-strict", - ) - with t[4]: - st.info(""" - **Quantification (ProteomicsLFQ):** - * **intThreshold**: Peak intensity threshold applied in seed detection. - * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR - * **proteinFDR**: Protein FDR threshold (0.05=5%). - """) - self.ui.input_TOPP( - "IDMapper", - custom_defaults={ - "threads": 8, - "debug": 0, - }, - tool_instance_name="IDMapper-TMT", - ) - with t[5]: - self.ui.input_TOPP( - "FileMerger", - custom_defaults={ - "in_type": "consensusXML", - "append_method": "append_cols", - "annotate_file_origin": True, - "threads": 8, - }, - flag_parameters=["annotate_file_origin"], - tool_instance_name="FileMerger-TMT", - ) - with t[6]: - self.ui.input_TOPP( - "ProteinInference", - custom_defaults={ - "threads": 8, - "picked_decoy_string": "DECOY_", - "picked_fdr": "true", - "protein_fdr": "true", - "Algorithm:use_shared_peptides": "true", - "Algorithm:annotate_indistinguishable_groups": "true", - "Algorithm:score_type": "PEP", - "Algorithm:score_aggregation_method": "best", - "Algorithm:min_peptides_per_protein": 1, - }, - tool_instance_name="ProteinInference-TMT", - ) - with t[7]: - # A single checkbox widget for workflow logic. - # self.ui.input_widget("run-python-script", False, "Run custom Python script") * - # Generate input widgets for a custom Python tool, located at src/python-tools. - # Parameters are specified within the file in the DEFAULTS dictionary. - # self.ui.input_python("example") * - self.ui.input_TOPP( - "IDFilter", - custom_defaults={ - "score:type_protein": "q-value", - "score:proteingroup": 0.01, - "score:psm": 0.01, - "delete_unreferenced_peptide_hits": True, - "remove_decoys": True - }, - flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], - tool_instance_name="IDFilter-lenient", - ) - with t[8]: - self.ui.input_TOPP( - "IDConflictResolver", - custom_defaults={ - "threads": 4, - }, - tool_instance_name="IDConflictResolver-TMT", - ) - - with t[9]: - self.ui.input_TOPP( - "ProteinQuantifier", - custom_defaults={ - "method": "top", - "top:N": 3, - "top:aggregate": "median", - "top:include_all": True, - "ratios": True, - "threads": 8, - "debug": 0, - }, - flag_parameters=["top:include_all", "ratios"], - tool_instance_name="ProteinQuantifier-TMT", - ) - with t[10]: - st.markdown("### πŸ§ͺ TMT Sample Group Assignment") - - # 1. Determine TMT type (e.g., tmt10plex, tmt16plex) - target_key = f"{self.parameter_manager.topp_param_prefix}IsobaricAnalyzer:1:type" - selected_tmt = st.session_state.get(target_key, "tmt12plex") - - if "tmt" in selected_tmt: - import re - # Extract the number to determine the plex count - num_plex_match = re.search(r'\d+', selected_tmt) - if num_plex_match: - num_plex = int(num_plex_match.group()) - all_channels = [f"sample{i+1}" for i in range(num_plex)] - - st.info( - "Enter a group name for each TMT channel.\n\n" - "Type **'skip'** for channels you wish to skip. (e.g., control, case, skip)" - ) - - # 2. Create an input_widget for each channel (automatically saved to params.json) - cols = st.columns(2) - for i, ch in enumerate(all_channels): - with cols[i % 2]: - self.ui.input_widget( - key=f"TMT-group-{ch}", - default="", - name=f"Group for {ch}", - widget_type="text", - help="Enter group name or 'skip' to ignore this channel.", - ) - - # 3. Read values from params.json and construct a dictionary in tmt_group_map format - # (This can be used later to filter DataFrames in subsequent logic) - self.params = self.parameter_manager.get_parameters_from_json() - - tmt_group_map = {} - for i, ch in enumerate(all_channels): - # Retrieve stored value (default is empty string) - group_val = self.params.get(f"TMT-group-{ch}", "") - tmt_group_map[str(i)] = group_val - - # For data inspection (remove if not needed) - if st.checkbox("Show current TMT mapping"): - st.json(tmt_group_map) - - # 4. Clean up parameters from unused/previous TMT settings - all_possible_channel_keys = {f"TMT-group-{ch}" for ch in all_channels} - orphaned_keys = [ - k for k in self.params.keys() - if k.startswith("TMT-group-") and k not in all_possible_channel_keys - ] - - if orphaned_keys: - for key in orphaned_keys: - del self.params[key] - self.parameter_manager.save_parameters() - - else: - st.warning("Please select a TMT type in the parameters first.") - def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -670,944 +350,639 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - current_mode = self.params.get("analysis-mode", "LFQ") - st.write(f"Current analysis mode: **{current_mode}**") - - if current_mode == "LFQ": - self.logger.log("βš™οΈ Running LFQ workflow") - - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "psm_filter" - quant_dir = results_dir / "quant_results" - - results_dir = Path(self.workflow_dir, "input-files") - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # ================================ - # 2️⃣ File path definitions (per sample) - # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) - - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") - - self.logger.log("πŸ”¬ Starting per-sample processing...") - - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "filter_results" + quant_dir = results_dir / "quant_results" + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # # ================================ + # # 2️⃣ File path definitions (per sample) + # # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + self.logger.log("πŸ”¬ Starting per-sample processing...") - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Peptide search complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(comet_results).exists(): - # st.error(f"CometAdapter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Peptide search complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False + + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + self.logger.log("βœ… Rescoring complete") - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False + + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Filtering complete") + self.logger.log("βœ… Filtering complete") - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") - else: - self.logger.log("Warning: Failed to build spectral library") + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) else: - self.logger.log("Warning: No PSMs converted for library generation") - - st.success(f"βœ“ {stem} identification completed") - - # # ================================ - # # 4️⃣ ProteomicsLFQ (cross-sample) - # # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") - - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") - - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) - - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) - - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) - - if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - }, - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… Quantification complete") - - # if not Path(quant_mztab).exists(): - # st.error("ProteomicsLFQ failed: mzTab not created") - # st.stop() - - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - st.write("πŸ“„ Generated files:") - st.write(f"- mzTab: {quant_mztab}") - st.write(f"- consensusXML: {quant_cxml}") - st.write(f"- MSstats CSV: {quant_msstats}") - - return True - else: - self.logger.log("βš™οΈ Running TMT workflow") - - results_dir = Path(self.workflow_dir, "results") - iso_dir = results_dir / "isobaric_consensusXML" - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - psm_filter_dir = results_dir / "psm_filter" - map_dir = results_dir / "idmapper" - merge_dir = results_dir / "merged" - protein_dir = results_dir / "protein" - msstats_dir = results_dir / "msstats" - quant_dir = results_dir / "quant_results" - - iso_consensus = [] - comet_results = [] - percolator_results = [] - psm_filtered = [] - mapped_ids = [] - - for d in [ - iso_dir, comet_dir, perc_dir, psm_filter_dir, - map_dir, merge_dir, protein_dir, msstats_dir, quant_dir - ]: - d.mkdir(parents=True, exist_ok=True) - - for mz in in_mzML: - stem = Path(mz).stem - iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) - psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) - mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) - - merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") - protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") - protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") - protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") - consensus_out = str(quant_dir / "openms_design_protein_openms.csv") - - # --- IsobaricAnalyzer --- - self.logger.log("🏷️ Running isobaric labeling analysis...") - with st.spinner("IsobaricAnalyzer"): - if not self.executor.run_topp( - "IsobaricAnalyzer", - { - "in": in_mzML, - "out": iso_consensus, - }, - tool_instance_name="IsobaricAnalyzer-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IsobaricAnalyzer complete") - - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - tool_instance_name="CometAdapter-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… CometAdapter complete") - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") + else: + self.logger.log("Warning: No PSMs converted for library generation") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + st.success(f"βœ“ {stem} identification completed") - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # ================================ + # 4️⃣ ProteomicsLFQ (cross-sample) + # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") - self.logger.log("βœ… Peptide search complete") + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - tool_instance_name="PercolatorAdapter-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) - - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) - - self.logger.log("βœ… PercolatorAdapter complete") - - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter"): if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": psm_filtered, - }, - tool_instance_name="IDFilter-strict" - ): + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "psmFDR": 0.5, + "proteinFDR": 0.5, + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + } + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… IDFilter-strict complete") - - # Build visualization cache for Filter results - for idxml_file in psm_filtered: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Quantification complete") + + # ====================================================== + # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) + # ====================================================== + workspace_path = Path(self.workflow_dir).parent + res = get_abundance_data(workspace_path) + if res is not None: + pivot_df, _, _ = res + self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") + self._run_go_enrichment(pivot_df, results_dir) + else: + st.warning("GO enrichment skipped: abundance data not available.") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + return True + + def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + self.logger.log("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + self.logger.log("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + self.logger.log("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + self.logger.log(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + self.logger.log(f"βœ… Plotly Figure generated") - # --- IDMapper --- - self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") - for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): - iso_stem = Path(iso).stem - with st.spinner(f"IDMapper ({iso_stem})"): - if not self.executor.run_topp( - "IDMapper", - { - "in": [iso], - "id": [psm], - "out": [mapped], - }, - tool_instance_name="IDMapper-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDMapper complete") + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) - # --- FileMerger --- - self.logger.log("πŸ”— Merging mapped consensus files...") - with st.spinner("FileMerger"): - if not self.executor.run_topp( - "FileMerger", - { - "in": mapped_ids, - "out": [merged_id], - }, - tool_instance_name="FileMerger-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… FileMerger complete") + return fig, df - # --- ProteinInference --- - self.logger.log("🧩 Running protein inference...") - with st.spinner("ProteinInference"): - if not self.executor.run_topp( - "ProteinInference", - { - "in": [merged_id], - "out": [protein_id], - }, - tool_instance_name="ProteinInference-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… ProteinInference complete") + go_results = {} - # --- IDFilter-lenient (Protein) --- - self.logger.log("πŸ”¬ Filtering proteins...") - with st.spinner("IDFilter (Protein)"): - if not self.executor.run_topp( - "IDFilter", - { - "in": [protein_id], - "out": [protein_filter], - }, - tool_instance_name="IDFilter-lenient" - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDFilter-lenient (Protein) complete") + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + self.logger.log(f"βœ… go_type generated") - # ================================ - # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) - # ================================ - self.logger.log("βš–οΈ Resolving ID conflicts...") - with st.spinner("IDConflictResolver"): - if not self.executor.run_topp( - "IDConflictResolver", - { - "in": [protein_filter], - "out": [protein_resolved], - }, - tool_instance_name="IDConflictResolver-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… IDConflictResolver complete") + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) - # ================================ - # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) - # ================================ - self.logger.log("πŸ“ Running protein quantification...") - with st.spinner("ProteinQuantifier"): - if not self.executor.run_topp( - "ProteinQuantifier", - { - "in": [protein_resolved], - "out": [consensus_out], - }, - tool_instance_name="ProteinQuantifier-TMT", - ): - self.logger.log("Workflow stopped due to error") - return False - self.logger.log("βœ… ProteinQuantifier complete") - self.logger.log("πŸ“„ Generating protein table...") - - self.logger.log("πŸŽ‰ WORKFLOW FINISHED") + import json + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + self.logger.log("βœ… GO enrichment analysis complete") + @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index 02d7bd4..db3e103 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -9,7 +9,6 @@ from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager from statsmodels.stats.multitest import multipletests -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -198,289 +197,111 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - - workflow_params = parameter_manager.get_parameters_from_json() - analysis_mode = workflow_params.get("analysis-mode", "LFQ") + if not quant_dir.exists(): + return None - if analysis_mode == "LFQ": - if not quant_dir.exists(): - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None + csv_file = csv_files[0] - csv_file = csv_files[0] + try: + df = pd.read_csv(csv_file) + except Exception: + return None - try: - df = pd.read_csv(csv_file) - except Exception: - return None + if df.empty: + return None - if df.empty: - return None + # Get group mapping from parameters + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if not group_map: + return None - if not group_map: - return None + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + df["Group"] = df["Reference"].map(group_map) + df = df.dropna(subset=["Group"]) - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + groups = sorted(df["Group"].unique()) - groups = sorted(df["Group"].unique()) + if len(groups) < 2: + return None - if len(groups) < 2: - return None + group1, group2 = groups[:2] - group1, group2 = groups[:2] + # Compute statistics per protein + stats_rows = [] + for protein, protein_df in df.groupby("ProteinName"): + g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values + g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + if len(g1_vals) < 2 or len(g2_vals) < 2: + pval = np.nan + else: + _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan - else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) + mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan + mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan + log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + stats_rows.append({ + "ProteinName": protein, + "log2FC": log2fc, + "p-value": pval, + }) - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + stats_df = pd.DataFrame(stats_rows) - stats_df = pd.DataFrame(stats_rows) - - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj - else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) - - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] - - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() - - return pivot_df, expr_df, group_map - - else: - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") - except Exception: - return None - - if df.empty: - return None - - # ratio column removal - df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] - - # exclude_indices = st.session_state.get("tmt_exclude_indices", []) - # group_map = st.session_state.get("tmt_group_map", {}) - # Get group mapping from parameters - parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") - params = parameter_manager.get_parameters_from_json() - group_map = {} - for key, value in params.items(): - if key.startswith("TMT-group-") and value: - # Extract the numeric part from keys like "TMT-group-sample1" - match = re.search(r'sample(\d+)', key) - if match: - # Subtract 1 to convert to a 0-based index (0, 1, 2...). - # If your samples are already 0-based, remove the -1 adjustment. - index = str(int(match.group(1)) - 1) - group_map[index] = value - - # 1. Extract keys labeled as "skip" from group_map as integer list - exclude_indices = [ - int(k) for k, v in group_map.items() if v.lower() == "skip" - ] - - # 2. Remove "skip" entries from group_map (keep only actual group info) - group_map = { - int(k): v for k, v in group_map.items() if v.lower() != "skip" + if not stats_df.empty: + mask = stats_df["p-value"].notna() + if mask.any(): + _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") + stats_df.loc[mask, "p-adj"] = p_adj + else: + stats_df["p-adj"] = np.nan + + # Order samples by group (group2 first, then group1) + sample_group_df = df[["Sample", "Group"]].drop_duplicates() + group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() + group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() + all_samples = group2_samples + group1_samples + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, } + pivot_list.append(row) - start_column_offset = 4 - - # st.write("exclude_indices:", exclude_indices) - # st.write("group_map:", group_map) - - if not group_map: - st.warning("⚠️ Group mapping information is missing. Please configure sample groups in the Setup page.") - return None - - if exclude_indices: - # st.write("Current columns:", df.columns.tolist()) - # st.write("Number of columns:", len(df.columns)) - # st.write("Exclude indices:", exclude_indices) - # st.write("Offset:", start_column_offset) - cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] - df_cleaned = df.drop(columns=cols_to_drop) - else: - df_cleaned = df.copy() - - if group_map: - # Create new row data (defaulting to empty strings) - # Create a list with the same length as the column order of df_cleaned - new_row = [""] * len(df_cleaned.columns) - new_row[0] = "Group" - - # Get the column names of the current dataframe as a list - current_cols = df_cleaned.columns.tolist() - original_cols = df.columns.tolist() - - for col_name in current_cols[start_column_offset:]: - # Check the original index position of this column - original_idx = original_cols.index(col_name) - start_column_offset - col_pos = current_cols.index(col_name) - new_row[col_pos] = group_map.get(original_idx, "NA") - - # Insert the row at the top of the dataframe - # Create a new DF and concatenate to prepend the row to existing data - group_df = pd.DataFrame([new_row], columns=df_cleaned.columns) - df_with_groups = pd.concat([group_df, df_cleaned], ignore_index=True) - - # drop_msg = f"{len(exclude_indices)} channels dropped" if exclude_indices else "No channels dropped" - # st.success(f"βœ… {drop_msg} and Group names have been inserted at the top of the data.") - - # st.write("### Data Preview with Group Information") - # st.dataframe(df_with_groups.head(10)) - - if group_map and len(set(group_map.values())) >= 2: - # Prepare data for calculation - # Extract group information from row 0 of df_with_groups (the newly added Group row) - # Actual sample data starts from the 5th column (index 4) - group_info_row = df_with_groups.iloc[0] - - # Get unique group names (excluding NA) - unique_groups = sorted([g for g in set(group_map.values()) if g != "NA"]) - g1_name, g2_name = unique_groups[0], unique_groups[1] - - # Extract numerical data for statistical calculation (from row 1 and column index 4 onwards) - # Convert to numeric type (to prevent calculation errors) - numeric_data = df_with_groups.iloc[1:, 4:].apply(pd.to_numeric, errors='coerce') - - # Column indexing by group - # Categorize columns based on the values in the Group row - g1_cols = [col for col in numeric_data.columns if group_info_row[col] == g1_name] - g2_cols = [col for col in numeric_data.columns if group_info_row[col] == g2_name] - - # Calculate log2FC and p-value for each row - def run_stats(row): - v1 = row[g1_cols].dropna() - v2 = row[g2_cols].dropna() - - # log2FC (Group2 / Group1) - m1, m2 = v1.mean(), v2.mean() - l2fc = np.log2(m2 / m1) if m1 > 0 and m2 > 0 else np.nan - - # p-value (T-test) - if len(v1) > 1 and len(v2) > 1: - _, pval = ttest_ind(v1, v2, equal_var=False) - else: - pval = np.nan - return pd.Series([l2fc, pval]) - - stats_results = numeric_data.apply(run_stats, axis=1) - stats_results.columns = ['log2FC', 'p-value'] - # Add Adjusted p-value (FDR) calculation - if not stats_results['p-value'].isna().all(): - # Select only rows that contain p-values - mask = stats_results['p-value'].notna() - # Apply Benjamini-Hochberg (BH) correction - _, p_adj, _, _ = multipletests(stats_results.loc[mask, 'p-value'], method='fdr_bh') - stats_results.loc[mask, 'p-adj'] = p_adj - else: - stats_results['p-adj'] = np.nan - - # Construct the final dataframe (Based on df_cleaned - excluding the group row) - # Insert calculation results into the 2nd and 3rd column positions - pivot_df = df_cleaned.copy() - pivot_df.insert(1, "log2FC", stats_results['log2FC'].values) - pivot_df.insert(2, "p-value", stats_results['p-value'].values) - pivot_df.insert(3, "p-adj", stats_results['p-adj'].values) - - # st.success(f"Analysis Complete: {g1_name} (n={len(g1_cols)}) vs {g2_name} (n={len(g2_cols)})") - - # Set the first column ('protein') of final_df as the index - protein_col = pivot_df.columns[0] - sample_cols = current_cols[start_column_offset:] # Identify actual sample column names - - # Select sample columns and create a matrix - expr_df = pivot_df.set_index(protein_col)[sample_cols] - - # Replace 0 with NaN (to prevent log transformation errors) - expr_df = expr_df.replace(0, np.nan) - - # Log2 transformation (data normalization) - expr_df = np.log2(expr_df + 1) - - # Remove proteins (rows) with any missing values - expr_df = expr_df.dropna() - - return pivot_df, expr_df, group_map - else: - st.warning("⚠️ At least two distinct groups are required for statistical analysis.") - else: - st.warning("⚠️ No group mapping information is set. Please check the Configure page.") - return None + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") + pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 6a587cd..86f265b 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -216,7 +216,7 @@ def read_stderr(): stdout_thread.join() stderr_thread.join() - def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool_instance_name: str = None) -> bool: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> bool: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -234,9 +234,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. - tool_instance_name (str, optional): A unique instance name for this tool - invocation, used for parameter lookup when multiple instances of the - same tool exist. If not provided, defaults to the tool name. Returns: bool: True if all commands succeeded, False if any failed. @@ -245,8 +242,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. """ - # Use tool_instance_name for parameter lookup, fall back to tool name - params_key = tool_instance_name if tool_instance_name else tool # check input: any input lists must be same length, other items can be a single string # e.g. input_mzML : [list of n mzML files], output_featureXML : [list of n featureXML files], input_database : database.tsv io_lengths = [len(v) for v in input_output.values() if len(v) > 1] @@ -266,13 +261,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool commands = [] - # Load merged parameters (_defaults + user overrides) for this tool instance - merged_params = self.parameter_manager.get_merged_params(params_key) - flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) - if not flag_map: - flag_map = st.session_state.get("_topp_flag_params", {}) - flag_params = set(flag_map.get(params_key, [])) - + # Load parameters for non-defaults + params = self.parameter_manager.get_parameters_from_json() # Construct commands for each process for i in range(n_processes): command = [tool] @@ -291,56 +281,35 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # standard case, files was a list of strings, take the file name at index else: command += [value[i]] - # Add merged TOPP tool parameters (_defaults + user overrides) - for k, v in merged_params.items(): - if k in flag_params: - # CLI flag: include "-k" only when enabled - if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} - else: - is_enabled = bool(v) - if is_enabled: - command += [f"-{k}"] - continue - # For non-flag parameters, skip entirely if empty. - # Note: 0 and 0.0 are valid values, so use explicit checks. - if v == "" or v is None: - continue - command += [f"-{k}"] - if isinstance(v, str) and "\n" in v: - command += v.split("\n") - elif isinstance(v, bool): - command += [str(v).lower()] - else: - command += [str(v)] + # Add non-default TOPP tool parameters + if tool in params.keys(): + for k, v in params[tool].items(): + command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): - if k in flag_params: - if isinstance(v, str): - is_enabled = v.lower() in {"true", "1", "yes", "on"} - else: - is_enabled = bool(v) - if is_enabled: - command += [f"-{k}"] - continue - if v == "" or v is None: - continue command += [f"-{k}"] - if isinstance(v, list): - command += [str(x) for x in v] - elif isinstance(v, bool): - command += [str(v).lower()] - else: - command += [str(v)] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) - for idx, cmd in enumerate(commands): - # Print list-form command joined into a single string for readability - print(f" πŸ”Ή Command {idx + 1}: {' '.join(cmd)}") - print("==========================================================\n") - + # check if a ini file has been written, if yes use it (contains custom defaults) + ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") + if ini_path.exists(): + command += ["-ini", str(ini_path)] # Run command(s) if len(commands) == 1: diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index 19e8700..b0c3626 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -129,30 +129,6 @@ def get_parameters_from_json(self) -> dict: except: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - - def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: - """ - Three-layer parameter merge: ini defaults < _defaults < user overrides. - - Args: - tool_instance_name: Instance name (or tool name) to look up in params.json. - ini_params: Base parameters from the .ini file. Optional β€” callers that - don't need the ini layer (e.g., run_topp, which passes -ini separately) - can omit this. - - Returns: - Merged dict with the effective value for each parameter. - """ - params = self.get_parameters_from_json() - defaults = params.get("_defaults", {}).get(tool_instance_name, {}) - user = params.get(tool_instance_name, {}) - - merged = {} - if ini_params: - merged.update(ini_params) - merged.update(defaults) - merged.update(user) - return merged def get_topp_parameters(self, tool: str) -> dict: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index befde2e..0af89aa 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -613,12 +613,10 @@ def input_TOPP( num_cols: int = 4, exclude_parameters: List[str] = [], include_parameters: List[str] = [], - flag_parameters: List[str] = [], display_tool_name: bool = True, display_subsections: bool = True, display_subsection_tabs: bool = False, custom_defaults: dict = {}, - tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -630,59 +628,33 @@ def input_TOPP( num_cols (int, optional): Number of columns to use for the layout. Defaults to 3. exclude_parameters (List[str], optional): List of parameter names to exclude from the widget. Defaults to an empty list. include_parameters (List[str], optional): List of parameter names to include in the widget. Defaults to an empty list. - flag_parameters (List[str], optional): List of parameter names that should - be treated as no-value CLI flags during command construction. display_tool_name (bool, optional): Whether to display the TOPP tool name. Defaults to True. display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. - tool_instance_name (str, optional): A unique instance name for this tool - invocation. Allows multiple instances of the same TOPP tool with - independent parameters (e.g., two IDFilter calls). If not provided, - defaults to topp_tool_name. The instance name is used for session - state keys and parameter storage, while topp_tool_name is used for - the actual tool executable and ini file creation. """ - # Default instance name to the tool name when not provided - if tool_instance_name is None: - tool_instance_name = topp_tool_name - - # Register instance-name β†’ real-tool-name mapping in session state - if "_topp_tool_instance_map" not in st.session_state: - st.session_state["_topp_tool_instance_map"] = {} - st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name - if "_topp_flag_params" not in st.session_state: - st.session_state["_topp_flag_params"] = {} - st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) - # Persist flag metadata so execution still sees it outside UI reruns/session context. - params = self.parameter_manager.get_parameters_from_json() - if "_flag_params" not in params: - params["_flag_params"] = {} - params["_flag_params"][tool_instance_name] = list(flag_parameters) - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: display_subsections = True - # Create pristine ini file (never mutated with custom defaults) + # write defaults ini files ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") + ini_existed = ini_file_path.exists() if not self.parameter_manager.create_ini(topp_tool_name): st.error(f"TOPP tool **'{topp_tool_name}'** not found.") return - - # Seed custom defaults into params.json under _defaults key - if custom_defaults: - params = self.parameter_manager.get_parameters_from_json() - if "_defaults" not in params: - params["_defaults"] = {} - params["_defaults"][tool_instance_name] = custom_defaults - with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: - json.dump(params, f, indent=4) - # Refresh self.params so widget resolution sees _defaults - self.params = self.parameter_manager.get_parameters_from_json() + if not ini_existed: + # update custom defaults if necessary + if custom_defaults: + param = poms.Param() + poms.ParamXMLFile().load(str(ini_file_path), param) + for key, value in custom_defaults.items(): + encoded_key = f"{topp_tool_name}:1:{key}".encode() + if encoded_key in param.keys(): + param.setValue(encoded_key, value) + poms.ParamXMLFile().store(str(ini_file_path), param) # read into Param object param = poms.Param() @@ -752,7 +724,6 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ":".join(key.decode().split(":")[:-1]) ), } - p["is_flag"] = (b"flag" in param.getTags(key)) # Parameter sections and subsections as string (e.g. "section:subsection") if display_subsections: p["sections"] = ":".join( @@ -760,18 +731,18 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ) params.append(p) - # Build ini_params dict for three-layer merge - ini_params = {} - for p in params: - name = p["key"].decode().split(":1:")[1] - ini_params[name] = p["value"] - - # Resolve effective values: ini < _defaults < user overrides - merged = self.parameter_manager.get_merged_params(tool_instance_name, ini_params=ini_params) + # for each parameter in params_decoded + # if a parameter with custom default value exists, use that value + # else check if the parameter is already in self.params, if yes take the value from self.params for p in params: name = p["key"].decode().split(":1:")[1] - if name in merged: - p["value"] = merged[name] + if topp_tool_name in self.params: + if name in self.params[topp_tool_name]: + p["value"] = self.params[topp_tool_name][name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] + elif name in custom_defaults: + p["value"] = custom_defaults[name] # Ensure list parameters stay as lists after loading from JSON # (JSON may store single-item lists as strings) if p["original_is_list"] and isinstance(p["value"], str): @@ -805,7 +776,7 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: # Display tool name if required if display_tool_name: - st.markdown(f"**{tool_instance_name}**") + st.markdown(f"**{topp_tool_name}**") tab_names = [k for k in param_sections.keys() if ":" not in k] tabs = None @@ -833,53 +804,23 @@ def display_TOPP_params(params: dict, num_cols): cols = st.columns(num_cols) i = 0 for p in params: - # get key and name – use tool_instance_name in session state key - key_str = p['key'].decode() - if tool_instance_name != topp_tool_name: - key_str = key_str.replace(f"{topp_tool_name}:1:", f"{tool_instance_name}:1:", 1) - key = f"{self.parameter_manager.topp_param_prefix}{key_str}" + # get key and name + key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" name = p["name"] try: # sometimes strings with newline, handle as list if isinstance(p["value"], str) and "\n" in p["value"]: p["value"] = p["value"].split("\n") - # no-value CLI flag parameters should be shown as checkboxes - if p.get("is_flag", False): - flag_default = p["value"] - if isinstance(flag_default, str): - flag_default = flag_default.lower() in {"true", "1", "yes", "on"} - else: - flag_default = bool(flag_default) - # Streamlit widget keys persist in session_state and can override - # updated custom_defaults. Normalize and seed key explicitly. - if key in st.session_state: - current = st.session_state[key] - if isinstance(current, str): - st.session_state[key] = current.lower() in {"true", "1", "yes", "on"} - else: - st.session_state[key] = bool(current) - else: - st.session_state[key] = flag_default - cols[i].selectbox( - name, - options=[True, False], - index=0 if st.session_state[key] else 1, - format_func=lambda x: "True" if x else "False", - help=p["description"], - key=key, - ) # bools - elif isinstance(p["value"], bool): - bool_value = ( - (p["value"] == "true") - if type(p["value"]) == str - else p["value"] - ) - cols[i].selectbox( + if isinstance(p["value"], bool): + cols[i].markdown("##") + cols[i].checkbox( name, - options=[True, False], - index=0 if bool_value else 1, - format_func=lambda x: "True" if x else "False", + value=( + (p["value"] == "true") + if type(p["value"]) == str + else p["value"] + ), help=p["description"], key=key, ) @@ -964,6 +905,7 @@ def on_multiselect_change(dk=display_key, tk=key): cols[i].error(f"Error in parameter **{p['name']}**.") print('Error parsing "' + p["name"] + '": ' + str(e)) + for section, params in param_sections.items(): if tabs is None: show_subsection_header(section, display_subsections) From b1dfb4d46bbb1635738a023adee560bb0c1c8115 Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 13:27:30 +0900 Subject: [PATCH 2/7] Sync workflow files to upstream/main (StreamlitUI, ParameterManager, CommandExecutor) --- src/workflow/CommandExecutor.py | 40 ++--- src/workflow/ParameterManager.py | 142 +++++++++++++-- src/workflow/StreamlitUI.py | 293 ++++++++++++++++++++++++++++--- 3 files changed, 409 insertions(+), 66 deletions(-) diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 86f265b..11bb148 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -5,7 +5,7 @@ import threading from pathlib import Path from .Logger import Logger -from .ParameterManager import ParameterManager +from .ParameterManager import ParameterManager, bool_param_paths_from_param_xml_ini import sys import importlib.util import json @@ -216,7 +216,7 @@ def read_stderr(): stdout_thread.join() stderr_thread.join() - def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> bool: + def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool_instance_name: str = None) -> bool: """ Constructs and executes commands for the specified tool OpenMS TOPP tool based on the given input and output configurations. Ensures that all input/output file lists @@ -234,6 +234,9 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b tool (str): The executable name or path of the tool. input_output (dict): A dictionary specifying the input/output parameter names (as key) and their corresponding file paths (as value). custom_params (dict): A dictionary of custom parameters to pass to the tool. + tool_instance_name (str, optional): A unique instance name for this tool + invocation, used for parameter lookup when multiple instances of the + same tool exist. If not provided, defaults to the tool name. Returns: bool: True if all commands succeeded, False if any failed. @@ -242,6 +245,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b ValueError: If the lengths of input/output file lists are inconsistent, except for single string inputs. """ + # Use tool_instance_name for parameter lookup, fall back to tool name + params_key = tool_instance_name if tool_instance_name else tool # check input: any input lists must be same length, other items can be a single string # e.g. input_mzML : [list of n mzML files], output_featureXML : [list of n featureXML files], input_database : database.tsv io_lengths = [len(v) for v in input_output.values() if len(v) > 1] @@ -261,8 +266,8 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b commands = [] - # Load parameters for non-defaults - params = self.parameter_manager.get_parameters_from_json() + # Load merged parameters (_defaults + user overrides) for this tool instance + merged_params = self.parameter_manager.get_merged_params(params_key) # Construct commands for each process for i in range(n_processes): command = [tool] @@ -281,20 +286,20 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b # standard case, files was a list of strings, take the file name at index else: command += [value[i]] - # Add non-default TOPP tool parameters - if tool in params.keys(): - for k, v in params[tool].items(): - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") - else: - command += [str(v)] + # Add merged TOPP tool parameters (_defaults + user overrides) + for k, v in merged_params.items(): + command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) + # Note: 0 and 0.0 are valid values, so use explicit check + if v != "" and v is not None: + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): command += [f"-{k}"] + # Skip only empty strings (pass flag with no value) # Note: 0 and 0.0 are valid values, so use explicit check if v != "" and v is not None: @@ -306,11 +311,6 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}) -> b command += ["-threads", str(threads_per_command)] commands.append(command) - # check if a ini file has been written, if yes use it (contains custom defaults) - ini_path = Path(self.parameter_manager.ini_dir, tool + ".ini") - if ini_path.exists(): - command += ["-ini", str(ini_path)] - # Run command(s) if len(commands) == 1: return self.run_command(commands[0]) diff --git a/src/workflow/ParameterManager.py b/src/workflow/ParameterManager.py index b0c3626..2838c1b 100644 --- a/src/workflow/ParameterManager.py +++ b/src/workflow/ParameterManager.py @@ -3,8 +3,52 @@ import shutil import subprocess import streamlit as st +import xml.etree.ElementTree as ET from pathlib import Path + +def bool_param_paths_from_param_xml_ini(ini_path: Path, tool_stem: str) -> set[str]: + """ + Return short parameter paths for every ```` in a ParamXML .ini file. + + Paths match the suffix after ``Tool:1:`` in pyOpenMS (e.g. ``algorithm:epd:masstrace_snr_filtering``). + """ + try: + root = ET.parse(ini_path).getroot() + except (ET.ParseError, OSError): + return set() + + def local_tag(el: ET.Element) -> str: + t = el.tag + return t.rsplit("}", 1)[-1] if isinstance(t, str) and "}" in t else str(t) + + out: set[str] = set() + + def walk(el: ET.Element, parts: tuple[str, ...]) -> None: + for ch in el: + lt = local_tag(ch) + if lt == "NODE": + nm = ch.get("name") or "" + walk(ch, parts + (nm,)) + elif lt == "ITEM" and (ch.get("type") or "").lower() == "bool": + nm = ch.get("name") or "" + segs = [p for p in parts if p] + if nm: + segs.append(nm) + if not segs: + continue + # Strip tool root NODE name and instance NODE "1" (not part of pyOpenMS short keys) + while segs and segs[0] in (tool_stem, "1"): + segs.pop(0) + if segs: + out.add(":".join(segs)) + + for ch in root: + if local_tag(ch) == "NODE": + walk(ch, ()) + return out + + class ParameterManager: """ Manages the parameters for a workflow, including saving parameters to a JSON file, @@ -29,6 +73,29 @@ def __init__(self, workflow_dir: Path, workflow_name: str = None): # Store workflow name for preset loading; default to directory stem if not provided self.workflow_name = workflow_name or workflow_dir.stem + def bool_pairs_session_key(self) -> str: + """Session state key holding a set of (tool name, param path) for bool TOPP params.""" + return f"{self.ini_dir.parent.stem}-topp-bool-pairs" + + def get_bool_param_pairs(self) -> set: + """Return the cached set of (tool, param path) bool params; empty set if none.""" + return st.session_state.get(self.bool_pairs_session_key(), set()) + + def _merge_bool_params_from_ini(self, tool: str) -> None: + """Load tool.ini (XML) and merge type=bool parameter paths into session_state.""" + ini_path = Path(self.ini_dir, f"{tool}.ini") + if not ini_path.exists(): + return + try: + sk = self.bool_pairs_session_key() + if sk not in st.session_state: + st.session_state[sk] = set() + for short in bool_param_paths_from_param_xml_ini(ini_path, tool): + st.session_state[sk].add((tool, short)) + except RuntimeError: + # No Streamlit session (e.g. plain `python` import) + pass + def create_ini(self, tool: str) -> bool: """ Create an ini file for a TOPP tool if it doesn't exist. @@ -41,11 +108,14 @@ def create_ini(self, tool: str) -> bool: """ ini_path = Path(self.ini_dir, tool + ".ini") if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return True try: subprocess.call([tool, "-write_ini", str(ini_path)]) except FileNotFoundError: return False + if ini_path.exists(): + self._merge_bool_params_from_ini(tool) return ini_path.exists() def save_parameters(self) -> None: @@ -65,7 +135,7 @@ def save_parameters(self) -> None: # Advanced parameters are only in session state if the view is active json_params = self.get_parameters_from_json() | json_params - # get a list of TOPP tools which are in session state + # get a list of TOPP tools (or tool instance names) which are in session state current_topp_tools = list( set( [ @@ -75,12 +145,16 @@ def save_parameters(self) -> None: ] ) ) - # for each TOPP tool, open the ini file + # Retrieve the instance-name β†’ real-tool-name mapping (set by input_TOPP) + tool_instance_map = st.session_state.get("_topp_tool_instance_map", {}) + # for each TOPP tool (or instance name), open the ini file for tool in current_topp_tools: - if not self.create_ini(tool): + # Resolve instance name to real tool name for create_ini / ini loading + real_tool = tool_instance_map.get(tool, tool) + if not self.create_ini(real_tool): # Could not create ini file - skip this tool continue - ini_path = Path(self.ini_dir, f"{tool}.ini") + ini_path = Path(self.ini_dir, f"{real_tool}.ini") if tool not in json_params: json_params[tool] = {} # load the param object @@ -92,19 +166,26 @@ def save_parameters(self) -> None: # Skip display keys used by multiselect widgets if key.endswith("_display"): continue - # get ini_key - ini_key = key.replace(self.topp_param_prefix, "").encode() + # get ini_key – map instance name back to real tool name + ini_key = key.replace(self.topp_param_prefix, "") + if tool != real_tool: + ini_key = ini_key.replace(f"{tool}:1:", f"{real_tool}:1:", 1) + ini_key = ini_key.encode() # get ini (default) value by ini_key ini_value = param.getValue(ini_key) is_list_param = isinstance(ini_value, list) - # check if value is different from default OR is an empty list parameter + # Effective default: _defaults value if present, else ini value + short_key = key.split(":1:")[1] + defaults = json_params.get("_defaults", {}).get(tool, {}) + default_value = defaults.get(short_key, ini_value) + # check if value is different from effective default OR is an empty list parameter if ( - (ini_value != value) - or (key.split(":1:")[1] in json_params[tool]) + (default_value != value) + or (short_key in json_params[tool]) or (is_list_param and not value) # Always save empty list params ): # store non-default value - json_params[tool][key.split(":1:")[1]] = value + json_params[tool][short_key] = value # Save to json file with open(self.params_file, "w", encoding="utf-8") as f: json.dump(json_params, f, indent=4) @@ -130,17 +211,44 @@ def get_parameters_from_json(self) -> dict: st.error("**ERROR**: Attempting to load an invalid JSON parameter file. Reset to defaults.") return {} - def get_topp_parameters(self, tool: str) -> dict: + def get_merged_params(self, tool_instance_name: str, ini_params: dict = None) -> dict: + """ + Three-layer parameter merge: ini defaults < _defaults < user overrides. + + Args: + tool_instance_name: Instance name (or tool name) to look up in params.json. + ini_params: Base parameters from the .ini file. Optional β€” callers that + don't need the ini layer (e.g., run_topp, which passes -ini separately) + can omit this. + + Returns: + Merged dict with the effective value for each parameter. + """ + params = self.get_parameters_from_json() + defaults = params.get("_defaults", {}).get(tool_instance_name, {}) + user = params.get(tool_instance_name, {}) + + merged = {} + if ini_params: + merged.update(ini_params) + merged.update(defaults) + merged.update(user) + return merged + + def get_topp_parameters(self, tool: str, tool_instance_name: str = None) -> dict: """ Get all parameters for a TOPP tool, merging defaults with user values. Args: - tool: Name of the TOPP tool (e.g., "CometAdapter") + tool: Name of the TOPP tool executable (e.g., "CometAdapter") + tool_instance_name: Optional instance name used for parameter storage + (e.g., "IDFilter_step1"). If not provided, defaults to tool name. Returns: Dict with parameter names as keys (without tool prefix) and their values. Returns empty dict if ini file doesn't exist. """ + instance_name = tool_instance_name or tool ini_path = Path(self.ini_dir, f"{tool}.ini") if not ini_path.exists(): return {} @@ -151,18 +259,14 @@ def get_topp_parameters(self, tool: str) -> dict: # Build dict from ini (extract short key names) prefix = f"{tool}:1:" - full_params = {} + ini_params = {} for key in param.keys(): key_str = key.decode() if isinstance(key, bytes) else str(key) if prefix in key_str: short_key = key_str.split(prefix, 1)[1] - full_params[short_key] = param.getValue(key) - - # Override with user-modified values from JSON - user_params = self.get_parameters_from_json().get(tool, {}) - full_params.update(user_params) + ini_params[short_key] = param.getValue(key) - return full_params + return self.get_merged_params(instance_name, ini_params=ini_params) def reset_to_default_parameters(self) -> None: """ diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 0af89aa..9c24dc2 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -23,6 +23,32 @@ from src.workflow._log_status import classify_log_outcome +def _mounted_data_root() -> Union[Path, None]: + """Return the validated mount root from the ``local_data_dir`` setting. + + The browser renders only when ``local_data_dir`` is an actual mount + point inside the container β€” i.e. the operator passed ``-v`` / + ``--bind`` / ``volumeMount`` to attach host data. Existence alone is + no longer sufficient because the image now pre-creates the path so + apptainer/singularity binds have a real attach target; without + ``os.path.ismount`` the browser would render an empty tree for every + user who didn't mount anything. + """ + settings = st.session_state.get("settings") or {} + raw = (settings.get("local_data_dir") or "").strip() + if not raw: + return None + try: + p = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + return None + if not p.is_dir(): + return None + if not os.path.ismount(p): + return None + return p + + class StreamlitUI: """ Provides an interface for Streamlit applications to handle file uploads, @@ -77,6 +103,8 @@ def upload_widget( c1, c2 = st.columns(2) c1.markdown("**Upload file(s)**") + mount_root = _mounted_data_root() if st.session_state.location == "online" else None + if st.session_state.location == "local": c2_text, c2_checkbox = c2.columns([1.5, 1], gap="large") c2_text.markdown("**OR add files from local folder**") @@ -247,7 +275,19 @@ def upload_widget( "This means that the original files will be used instead. " ) - if fallback and not any([f for f in Path(files_dir).iterdir() if f.name != "external_files.txt"]): + if mount_root is not None: + with c2: + self._mounted_drive_browser(key, name, file_types, files_dir, mount_root) + + external_files_path = Path(files_dir, "external_files.txt") + has_real_files = any( + p.name != "external_files.txt" for p in files_dir.iterdir() + ) + has_external_picks = external_files_path.exists() and any( + line.strip() and os.path.exists(line.strip()) + for line in external_files_path.read_text().splitlines() + ) + if fallback and not has_real_files and not has_external_picks: if isinstance(fallback, str): fallback = [fallback] for f in fallback: @@ -303,6 +343,179 @@ def upload_widget( elif not fallback: st.warning(f"No **{name}** files!") + def _resolve_browser_cwd(self, key: str, mount_root: Path) -> Path: + """Read cwd for this widget from session state, confine it to mount_root.""" + sess_key = f"mounted_cwd_{key}" + raw = st.session_state.get(sess_key, str(mount_root)) + try: + cwd = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError): + cwd = mount_root + if cwd != mount_root and mount_root not in cwd.parents: + cwd = mount_root + st.session_state[sess_key] = str(cwd) + return cwd + + def _mounted_drive_browser( + self, + key: str, + name: str, + file_types: List[str], + files_dir: Path, + mount_root: Path, + ) -> None: + """Render a tree browser for a mounted host directory. + + Selected files are referenced in place via ``external_files.txt`` β€” + the same mechanism the offline tkinter flow uses. + """ + external_files = Path(files_dir, "external_files.txt") + if not external_files.exists(): + external_files.touch() + + cwd = self._resolve_browser_cwd(key, mount_root) + sess_cwd_key = f"mounted_cwd_{key}" + + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + + with st.container(border=True): + st.markdown( + f"**Add {name} files from mounted directory** " + f"`{mount_root}`" + ) + + # Breadcrumbs: compact tertiary buttons separated by Β», + # with a right-aligned Parent button. + try: + rel = cwd.relative_to(mount_root) + segments = [mount_root.name] + list(rel.parts) if rel.parts else [mount_root.name] + except ValueError: + segments = [mount_root.name] + n = len(segments) + ratios: List[float] = [] + for i in range(n): + ratios.append(max(len(segments[i]), 3)) + if i < n - 1: + ratios.append(1) + ratios.append(20) # flexible spacer + ratios.append(6) # parent button slot + crumb_cols = st.columns(ratios, vertical_alignment="center") + col_idx = 0 + for i, seg in enumerate(segments): + target = mount_root.joinpath(*segments[1 : i + 1]) if i > 0 else mount_root + if crumb_cols[col_idx].button( + seg, + key=f"crumb_{key}_{i}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(target) + st.rerun(scope="fragment") + col_idx += 1 + if i < n - 1: + crumb_cols[col_idx].markdown( + "Β»", + unsafe_allow_html=True, + ) + col_idx += 1 + # spacer column + col_idx += 1 + if cwd != mount_root: + if crumb_cols[col_idx].button( + "⬆ Parent", + key=f"mounted_parent_{key}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(cwd.parent) + st.rerun(scope="fragment") + + try: + entries = sorted( + (p for p in cwd.iterdir() if not p.name.startswith(".")), + key=lambda p: (not p.is_dir(), p.name.lower()), + ) + except PermissionError: + st.error(f"Permission denied reading `{cwd}`.") + return + + def _is_match(p: Path) -> bool: + return any(p.name.endswith(f".{ft}") for ft in file_types) + + subdirs = [p for p in entries if p.is_dir() and not _is_match(p)] + bundled = [p for p in entries if p.is_dir() and _is_match(p)] + files = [p for p in entries if p.is_file() and _is_match(p)] + + for d in subdirs: + indent, body = st.columns([1, 60], vertical_alignment="center") + if body.button( + f"πŸ“‚ {d.name}/", + key=f"mounted_dir_{key}_{d.name}", + type="tertiary", + ): + st.session_state[sess_cwd_key] = str(d) + st.rerun(scope="fragment") + + selectable = bundled + files + selected_paths: List[str] = [] + for f in selectable: + cb_key = f"mounted_pick_{key}_{f}" + size_label = "" + if f.is_file(): + try: + size_mb = f.stat().st_size / (1024 * 1024) + size_label = f" Β· {size_mb:.1f} MB" + except OSError: + pass + icon = "πŸ—‚οΈ" if f.is_dir() else "πŸ“„" + if st.checkbox( + f"{icon} {f.name}{size_label}", + key=cb_key, + ): + selected_paths.append(str(f)) + + if not subdirs and not selectable: + st.info( + f"No subdirectories or files matching " + f"**{', '.join('.' + ft for ft in file_types)}** here." + ) + + count = len(selected_paths) + if st.button( + f"βž• Add {count} selected {name} file(s)" if count else f"βž• Add selected {name} file(s)", + key=f"mounted_add_{key}", + type="primary", + use_container_width=True, + disabled=count == 0, + ): + existing = set( + line.strip() + for line in external_files.read_text().splitlines() + if line.strip() + ) + added = 0 + with open(external_files, "a") as fh: + for p in selected_paths: + if p not in existing: + fh.write(f"{p}\n") + existing.add(p) + added += 1 + # Clear the checkboxes by removing their session keys. + for f in selectable: + st.session_state.pop(f"mounted_pick_{key}_{f}", None) + st.success(f"Added {added} file(s) from `{cwd}`.") + st.rerun(scope="fragment") + def select_input_file( self, key: str, @@ -617,6 +830,7 @@ def input_TOPP( display_subsections: bool = True, display_subsection_tabs: bool = False, custom_defaults: dict = {}, + tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's @@ -632,29 +846,43 @@ def input_TOPP( display_subsections (bool, optional): Whether to split parameters into subsections based on the prefix. Defaults to True. display_subsection_tabs (bool, optional): Whether to display main subsections in separate tabs (if more than one main section). Defaults to False. custom_defaults (dict, optional): Dictionary of custom defaults to use. Defaults to an empty dict. + tool_instance_name (str, optional): A unique instance name for this tool + invocation. Allows multiple instances of the same TOPP tool with + independent parameters (e.g., two IDFilter calls). If not provided, + defaults to topp_tool_name. The instance name is used for session + state keys and parameter storage, while topp_tool_name is used for + the actual tool executable and ini file creation. """ + # Default instance name to the tool name when not provided + if tool_instance_name is None: + tool_instance_name = topp_tool_name + + # Register instance-name β†’ real-tool-name mapping in session state + if "_topp_tool_instance_map" not in st.session_state: + st.session_state["_topp_tool_instance_map"] = {} + st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: display_subsections = True - # write defaults ini files + # Create pristine ini file (never mutated with custom defaults) ini_file_path = Path(self.parameter_manager.ini_dir, f"{topp_tool_name}.ini") - ini_existed = ini_file_path.exists() if not self.parameter_manager.create_ini(topp_tool_name): st.error(f"TOPP tool **'{topp_tool_name}'** not found.") return - if not ini_existed: - # update custom defaults if necessary - if custom_defaults: - param = poms.Param() - poms.ParamXMLFile().load(str(ini_file_path), param) - for key, value in custom_defaults.items(): - encoded_key = f"{topp_tool_name}:1:{key}".encode() - if encoded_key in param.keys(): - param.setValue(encoded_key, value) - poms.ParamXMLFile().store(str(ini_file_path), param) + + # Seed custom defaults into params.json under _defaults key + if custom_defaults: + params = self.parameter_manager.get_parameters_from_json() + if "_defaults" not in params: + params["_defaults"] = {} + params["_defaults"][tool_instance_name] = custom_defaults + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as f: + json.dump(params, f, indent=4) + # Refresh self.params so widget resolution sees _defaults + self.params = self.parameter_manager.get_parameters_from_json() # read into Param object param = poms.Param() @@ -731,18 +959,18 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: ) params.append(p) - # for each parameter in params_decoded - # if a parameter with custom default value exists, use that value - # else check if the parameter is already in self.params, if yes take the value from self.params + # Build ini_params dict for three-layer merge + ini_params = {} for p in params: name = p["key"].decode().split(":1:")[1] - if topp_tool_name in self.params: - if name in self.params[topp_tool_name]: - p["value"] = self.params[topp_tool_name][name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] - elif name in custom_defaults: - p["value"] = custom_defaults[name] + ini_params[name] = p["value"] + + # Resolve effective values: ini < _defaults < user overrides + merged = self.parameter_manager.get_merged_params(tool_instance_name, ini_params=ini_params) + for p in params: + name = p["key"].decode().split(":1:")[1] + if name in merged: + p["value"] = merged[name] # Ensure list parameters stay as lists after loading from JSON # (JSON may store single-item lists as strings) if p["original_is_list"] and isinstance(p["value"], str): @@ -776,7 +1004,7 @@ def _matches_parameter(pattern: str, key: bytes) -> bool: # Display tool name if required if display_tool_name: - st.markdown(f"**{topp_tool_name}**") + st.markdown(f"**{tool_instance_name}**") tab_names = [k for k in param_sections.keys() if ":" not in k] tabs = None @@ -804,8 +1032,11 @@ def display_TOPP_params(params: dict, num_cols): cols = st.columns(num_cols) i = 0 for p in params: - # get key and name - key = f"{self.parameter_manager.topp_param_prefix}{p['key'].decode()}" + # get key and name – use tool_instance_name in session state key + key_str = p['key'].decode() + if tool_instance_name != topp_tool_name: + key_str = key_str.replace(f"{topp_tool_name}:1:", f"{tool_instance_name}:1:", 1) + key = f"{self.parameter_manager.topp_param_prefix}{key_str}" name = p["name"] try: # sometimes strings with newline, handle as list @@ -1377,7 +1608,8 @@ def remove_full_paths(d: dict) -> dict: general = {} for k, v in params.items(): - # skip if v is a file path + if k == "_defaults": + continue if isinstance(v, dict): topp[k] = v elif ".py" in k: @@ -1388,6 +1620,13 @@ def remove_full_paths(d: dict) -> dict: else: general[k] = v + # Merge _defaults into topp so summary shows custom defaults + user overrides + defaults = params.get("_defaults", {}) + for tool_name, default_vals in defaults.items(): + if tool_name not in topp: + topp[tool_name] = {} + topp[tool_name] = {**default_vals, **topp.get(tool_name, {})} + markdown = [] def dict_to_markdown(d: dict): From c12bc1cfbcd2d158f9d2984cbdbe435dbbbc421b Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Wed, 15 Jul 2026 15:04:48 +0900 Subject: [PATCH 3/7] Add support for boolean CLI flag parameters in TOPP tools Allows input_TOPP() to designate parameters as flags (present/absent) instead of key-value pairs, persisted to params.json and session_state so run_topp() can correctly build the command line. --- src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++---------- src/workflow/StreamlitUI.py | 13 ++++++++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py index 11bb148..042c5e1 100644 --- a/src/workflow/CommandExecutor.py +++ b/src/workflow/CommandExecutor.py @@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool # Load merged parameters (_defaults + user overrides) for this tool instance merged_params = self.parameter_manager.get_merged_params(params_key) + + # Load flag parameter names: params.json takes priority (survives session restart), + # session_state is the live fallback during the current session. + flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) + if not flag_map: + flag_map = st.session_state.get("_topp_flag_params", {}) + flag_params: set = set(flag_map.get(params_key, [])) + # Construct commands for each process for i in range(n_processes): command = [tool] @@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool command += [value[i]] # Add merged TOPP tool parameters (_defaults + user overrides) for k, v in merged_params.items(): - command += [f"-{k}"] - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, str) and "\n" in v: - command += v.split("\n") + if k in flag_params: + # CLI flag: include "-k" only when truthy, omit when false + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + # Regular parameter: skip empty/None, append value otherwise + if v == "" or v is None: + continue + command += [f"-{k}"] + if isinstance(v, str) and "\n" in v: + command += v.split("\n") + else: + command += [str(v)] # Add custom parameters for k, v in custom_params.items(): - command += [f"-{k}"] - - # Skip only empty strings (pass flag with no value) - # Note: 0 and 0.0 are valid values, so use explicit check - if v != "" and v is not None: - if isinstance(v, list): - command += [str(x) for x in v] + if k in flag_params: + if isinstance(v, str): + is_enabled = v.lower() == "true" else: - command += [str(v)] + is_enabled = bool(v) + if is_enabled: + command += [f"-{k}"] + continue + if v == "" or v is None: + continue + command += [f"-{k}"] + if isinstance(v, list): + command += [str(x) for x in v] + else: + command += [str(v)] # Add threads parameter for TOPP tools command += ["-threads", str(threads_per_command)] commands.append(command) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 9c24dc2..4bd53b1 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -826,6 +826,7 @@ def input_TOPP( num_cols: int = 4, exclude_parameters: List[str] = [], include_parameters: List[str] = [], + flag_parameters: List[str] = [], display_tool_name: bool = True, display_subsections: bool = True, display_subsection_tabs: bool = False, @@ -862,6 +863,18 @@ def input_TOPP( st.session_state["_topp_tool_instance_map"] = {} st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + # Persist flag_parameters to session_state and params.json so run_topp + # can skip appending a value for these boolean CLI flags. + if "_topp_flag_params" not in st.session_state: + st.session_state["_topp_flag_params"] = {} + st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) + _fp = self.parameter_manager.get_parameters_from_json() + if "_flag_params" not in _fp: + _fp["_flag_params"] = {} + _fp["_flag_params"][tool_instance_name] = list(flag_parameters) + with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: + json.dump(_fp, _f, indent=4) + if not display_subsections: display_subsection_tabs = False if display_subsection_tabs: From af57d216085cdae53af46d02cd3b44979c7ac09f Mon Sep 17 00:00:00 2001 From: jkvision1101 Date: Thu, 16 Jul 2026 14:45:41 +0900 Subject: [PATCH 4/7] InputTOPP: add reactive option and split impl/fragment for conditional UI --- pr-397.patch | 119 ++++++++++++++++++++++++++++++++++++ src/WorkflowTest.py | 8 ++- src/workflow/StreamlitUI.py | 74 +++++++++++++++++++++- 3 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 pr-397.patch diff --git a/pr-397.patch b/pr-397.patch new file mode 100644 index 0000000..bb97819 --- /dev/null +++ b/pr-397.patch @@ -0,0 +1,119 @@ +From 268348f6da9809750e3116535f4f3fe9defc7ab1 Mon Sep 17 00:00:00 2001 +From: Yoo HoJun +Date: Wed, 15 Jul 2026 15:04:48 +0900 +Subject: [PATCH] Add support for boolean CLI flag parameters in TOPP tools + +Allows input_TOPP() to designate parameters as flags (present/absent) +instead of key-value pairs, persisted to params.json and session_state +so run_topp() can correctly build the command line. +--- + src/workflow/CommandExecutor.py | 53 +++++++++++++++++++++++---------- + src/workflow/StreamlitUI.py | 13 ++++++++ + 2 files changed, 51 insertions(+), 15 deletions(-) + +diff --git a/src/workflow/CommandExecutor.py b/src/workflow/CommandExecutor.py +index 11bb1486..042c5e11 100644 +--- a/src/workflow/CommandExecutor.py ++++ b/src/workflow/CommandExecutor.py +@@ -268,6 +268,14 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool + + # Load merged parameters (_defaults + user overrides) for this tool instance + merged_params = self.parameter_manager.get_merged_params(params_key) ++ ++ # Load flag parameter names: params.json takes priority (survives session restart), ++ # session_state is the live fallback during the current session. ++ flag_map = self.parameter_manager.get_parameters_from_json().get("_flag_params", {}) ++ if not flag_map: ++ flag_map = st.session_state.get("_topp_flag_params", {}) ++ flag_params: set = set(flag_map.get(params_key, [])) ++ + # Construct commands for each process + for i in range(n_processes): + command = [tool] +@@ -288,25 +296,40 @@ def run_topp(self, tool: str, input_output: dict, custom_params: dict = {}, tool + command += [value[i]] + # Add merged TOPP tool parameters (_defaults + user overrides) + for k, v in merged_params.items(): +- command += [f"-{k}"] +- # Skip only empty strings (pass flag with no value) +- # Note: 0 and 0.0 are valid values, so use explicit check +- if v != "" and v is not None: +- if isinstance(v, str) and "\n" in v: +- command += v.split("\n") ++ if k in flag_params: ++ # CLI flag: include "-k" only when truthy, omit when false ++ if isinstance(v, str): ++ is_enabled = v.lower() == "true" + else: +- command += [str(v)] ++ is_enabled = bool(v) ++ if is_enabled: ++ command += [f"-{k}"] ++ continue ++ # Regular parameter: skip empty/None, append value otherwise ++ if v == "" or v is None: ++ continue ++ command += [f"-{k}"] ++ if isinstance(v, str) and "\n" in v: ++ command += v.split("\n") ++ else: ++ command += [str(v)] + # Add custom parameters + for k, v in custom_params.items(): +- command += [f"-{k}"] +- +- # Skip only empty strings (pass flag with no value) +- # Note: 0 and 0.0 are valid values, so use explicit check +- if v != "" and v is not None: +- if isinstance(v, list): +- command += [str(x) for x in v] ++ if k in flag_params: ++ if isinstance(v, str): ++ is_enabled = v.lower() == "true" + else: +- command += [str(v)] ++ is_enabled = bool(v) ++ if is_enabled: ++ command += [f"-{k}"] ++ continue ++ if v == "" or v is None: ++ continue ++ command += [f"-{k}"] ++ if isinstance(v, list): ++ command += [str(x) for x in v] ++ else: ++ command += [str(v)] + # Add threads parameter for TOPP tools + command += ["-threads", str(threads_per_command)] + commands.append(command) +diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py +index 9c24dc2c..4bd53b18 100644 +--- a/src/workflow/StreamlitUI.py ++++ b/src/workflow/StreamlitUI.py +@@ -826,6 +826,7 @@ def input_TOPP( + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], ++ flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, +@@ -862,6 +863,18 @@ def input_TOPP( + st.session_state["_topp_tool_instance_map"] = {} + st.session_state["_topp_tool_instance_map"][tool_instance_name] = topp_tool_name + ++ # Persist flag_parameters to session_state and params.json so run_topp ++ # can skip appending a value for these boolean CLI flags. ++ if "_topp_flag_params" not in st.session_state: ++ st.session_state["_topp_flag_params"] = {} ++ st.session_state["_topp_flag_params"][tool_instance_name] = list(flag_parameters) ++ _fp = self.parameter_manager.get_parameters_from_json() ++ if "_flag_params" not in _fp: ++ _fp["_flag_params"] = {} ++ _fp["_flag_params"][tool_instance_name] = list(flag_parameters) ++ with open(self.parameter_manager.params_file, "w", encoding="utf-8") as _f: ++ json.dump(_fp, _f, indent=4) ++ + if not display_subsections: + display_subsection_tabs = False + if display_subsection_tabs: diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 4abbf92..2af9bda 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -2,8 +2,8 @@ from pathlib import Path import pandas as pd import plotly.express as px -from streamlit_plotly_events import plotly_events -from pyopenms import IdXMLFile +#from streamlit_plotly_events import plotly_events +#from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +14,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -from openms_insight import Table, Heatmap, LinePlot, SequenceView +#from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -129,6 +129,7 @@ def configure(self) -> None: "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", }, + flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, exclude_parameters=["second_enzyme"], ) @@ -152,6 +153,7 @@ def configure(self) -> None: "score_type": "pep", "post_processing_tdc": "true", }, + flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, exclude_parameters=["out_type"], ) diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index 4bd53b1..77ca01d 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -819,7 +819,6 @@ def format_files(input: Any) -> List[str]: self.parameter_manager.save_parameters() - @st.fragment def input_TOPP( self, topp_tool_name: str, @@ -832,6 +831,79 @@ def input_TOPP( display_subsection_tabs: bool = False, custom_defaults: dict = {}, tool_instance_name: str = None, + reactive: bool = False, + ) -> None: + """ + Wrapper for TOPP parameter input. When `reactive` is True the + implementation is rendered directly in the parent context so changes + trigger a parent re-render; otherwise the widgets are rendered inside + a `st.fragment` to isolate reruns for performance. + """ + if reactive: + return self._input_TOPP_impl( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + return self._input_TOPP_fragmented( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + + @st.fragment + def _input_TOPP_fragmented( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, + ) -> None: + return self._input_TOPP_impl( + topp_tool_name=topp_tool_name, + num_cols=num_cols, + exclude_parameters=exclude_parameters, + include_parameters=include_parameters, + flag_parameters=flag_parameters, + display_tool_name=display_tool_name, + display_subsections=display_subsections, + display_subsection_tabs=display_subsection_tabs, + custom_defaults=custom_defaults, + tool_instance_name=tool_instance_name, + ) + + def _input_TOPP_impl( + self, + topp_tool_name: str, + num_cols: int = 4, + exclude_parameters: List[str] = [], + include_parameters: List[str] = [], + flag_parameters: List[str] = [], + display_tool_name: bool = True, + display_subsections: bool = True, + display_subsection_tabs: bool = False, + custom_defaults: dict = {}, + tool_instance_name: str = None, ) -> None: """ Generates input widgets for TOPP tool parameters dynamically based on the tool's From 7c5a2250d0e79ae796adcea87c4950b63a0ed12b Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 5/7] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 14 files changed, 2971 insertions(+), 878 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map From 9c1df80a7386684f97800e40250ba153f1ba8392 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 6/7] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ requirements.txt | 8 +- src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 15 files changed, 2978 insertions(+), 879 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aac2879..5d20693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -142,6 +142,11 @@ scipy scikit-learn openms-insight>=0.1.13 polars>=1.0.0 +# Forces polars to prefer its most CPU-compatible native runtime. +# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32) +# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some +# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops. +polars-runtime-compat cython easypqp>=0.1.34 pyprophet>=2.2.0 @@ -149,4 +154,5 @@ mygene # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels \ No newline at end of file +statsmodels +polars \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map From e7dd66b9ec8666633de6dac4edccefc8722d4137 Mon Sep 17 00:00:00 2001 From: Yoo HoJun Date: Thu, 16 Jul 2026 15:15:14 +0900 Subject: [PATCH 7/7] Add LFQ/TMT workflow mode split and downstream analysis pages Split WorkflowTest configure() into separate LFQ and TMT tool tabs, and add new preprocessing/analysis pages (filtering, normalization, imputation, statistical testing, GO enrichment, clustered heatmap, pathway analysis) built on openms_insight engine functions. --- app.py | 20 +- content/enrichment.py | 141 ++ content/filtering.py | 173 +++ content/imputation.py | 145 ++ content/normalization.py | 242 ++++ content/results_abundance.py | 138 +- content/results_heatmap.py | 79 +- content/results_heatmap_clustered.py | 107 ++ content/results_pathway_analysis.py | 258 ++++ content/results_pca.py | 177 ++- content/results_proteomicslfq.py | 5 +- content/results_volcano.py | 91 +- content/statistical.py | 165 +++ requirements.txt | 8 +- src/WorkflowTest.py | 1841 +++++++++++++++++--------- src/common/results_helpers.py | 287 ++-- 16 files changed, 2995 insertions(+), 882 deletions(-) create mode 100644 content/enrichment.py create mode 100644 content/filtering.py create mode 100644 content/imputation.py create mode 100644 content/normalization.py create mode 100644 content/results_heatmap_clustered.py create mode 100644 content/results_pathway_analysis.py create mode 100644 content/statistical.py diff --git a/app.py b/app.py index 194d857..e76dcd8 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,10 @@ +import os +# Polars' default (CPU-core-count-sized) native thread pool access-violation +# crashes the whole process on some high-core-count Windows machines when +# invoked from Streamlit's script-runner thread. Must be set before polars +# is imported anywhere (including transitively via openms_insight). +os.environ.setdefault("POLARS_MAX_THREADS", "1") + import streamlit as st from pathlib import Path import json @@ -23,12 +30,19 @@ st.Page(Path("content", "results_rescoring.py"), title="Rescoring", icon="πŸ“ˆ"), st.Page(Path("content", "results_filtered.py"), title="Filtered PSMs", icon="🎯"), st.Page(Path("content", "results_abundance.py"), title="Abundance", icon="πŸ“‹"), + + ], + "Differential Protein Analysis": [ + st.Page(Path("content", "filtering.py"), title="Filtering", icon="🧹"), + st.Page(Path("content", "imputation.py"), title="Imputation", icon="🩹"), + st.Page(Path("content", "normalization.py"), title="Normalization", icon="βš–οΈ"), + st.Page(Path("content", "statistical.py"), title="Statistical", icon="πŸ”’"), st.Page(Path("content", "results_volcano.py"), title="Volcano", icon="πŸŒ‹"), st.Page(Path("content", "results_pca.py"), title="PCA", icon="πŸ“Š"), st.Page(Path("content", "results_heatmap.py"), title="Heatmap", icon="πŸ”₯"), - st.Page(Path("content", "results_library.py"), title="Spectral Library", icon="πŸ“š"), - st.Page(Path("content", "results_proteomicslfq.py"), title="Proteomics LFQ", icon="πŸ§ͺ"), - ], + st.Page(Path("content", "results_heatmap_clustered.py"), title="Clustered Heatmap", icon="🧬"), + st.Page(Path("content", "enrichment.py"), title="Pathway Analysis", icon="πŸ“‰"), + ] } pg = st.navigation(pages) diff --git a/content/enrichment.py b/content/enrichment.py new file mode 100644 index 0000000..62fc9fa --- /dev/null +++ b/content/enrichment.py @@ -0,0 +1,141 @@ +"""Pathway Analysis Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column +# Import GO Enrichment modules from openms_insight engine +from openms_insight.analysis.enrichment import calculate_go_enrichment + +params = page_setup() +st.title("GO Enrichment Analysis") + +st.markdown( + """ +Identify overrepresented biological themes (BP, CC, MF) within your differentially expressed protein features using MyGene.info and Fisher's Exact Test. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# --- STEP 1: Upstream Statistics Checkpoint --- +if ( + "statistics_df" in st.session_state + and st.session_state["statistics_df"] is not None +): + final_statistics_report = st.session_state["statistics_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using analyzed matrices from the **Statistical Inference** step." + ) +else: + st.warning( + "⚠️ **Missing Prerequisites**: Statistical inference data not detected. Please run hypothesis testing first." + ) + st.page_link( + "content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬" + ) + st.stop() + +# --- STEP 2: Preprocessing Mapping Key Configuration --- +# Identify target identifier columns dynamically +abundance_result = get_abundance_data(st.session_state["workspace"]) +id_col = get_id_column(st.session_state["workspace"], abundance_result[0]) if abundance_result else "ProteinName" +if id_col not in final_statistics_report.columns: + st.error(f"❌ Structural Error: Column '{id_col}' is missing from the active matrix context.") + st.stop() + +# --- SECTION 1: Parameter Setup & Dynamic Cutoff Labels --- +st.subheader("Configure Enrichment Thresholds") + +# Check if target p-value should be adjusted or raw based on previous selections (Fallback safely to 'p-adj') +target_p_col = "p-adj" if "p-adj" in final_statistics_report.columns else "p-value" +p_label = ( + "Adjusted P-value (p-adj) Cutoff" + if target_p_col == "p-adj" + else "Raw P-value (p-value) Cutoff" +) + +ui_go_col1, ui_go_col2 = st.columns(2) + +with ui_go_col1: + p_cutoff = st.number_input( + f"πŸ”¬ {p_label}", + min_value=0.0001, + max_value=1.0, + value=0.05, + step=0.01, + format="%.4f", + help="Proteins with significance metrics below this value are mapped to the foreground cohort.", + ) + +with ui_go_col2: + fc_cutoff = st.number_input( + "πŸ“ˆ Absolute Difference Cutoff (|log2FC|)", + min_value=0.0, + max_value=10.0, + value=1.0, + step=0.1, + format="%.2f", + help="Proteins with absolute log2 fold change greater than or equal to this threshold will be selected.", + ) + +# --- SECTION 2: Execution and Interactive View Charts --- +st.markdown("
", unsafe_allow_html=True) +if st.button("πŸš€ Run GO Enrichment Analysis", type="primary", key="run_go_analysis"): + + with st.spinner("Querying MyGene.info API & executing hyper-geometric calculation loops..."): + # Convert internal pandas DataFrame to openms_insight Polars DataFrame expectation + stats_pl = pl.from_pandas(final_statistics_report) + + status, output = calculate_go_enrichment( + final_report=stats_pl, + id_col=id_col, + target_p_col=target_p_col, + p_cutoff=p_cutoff, + fc_cutoff=fc_cutoff, + ) + + # Route response structures based on analysis output status code + if status == "empty_data": + st.error("❌ No valid statistical rows found containing standard columns to run GO alignment.") + + elif status == "insufficient_proteins": + st.warning( + f"⚠️ Not enough significant proteins found to construct target datasets. " + f"(Criteria: {target_p_col} < {p_cutoff:.4f}, |log2FC| β‰₯ {fc_cutoff:.2f})." + ) + st.info(f"πŸ’‘ Found significant proteins count: **{output}**. Try relaxing your p-value or log2FC filters.") + + elif status == "success": + st.success("β­• GO Enrichment Analysis completed successfully!") + + # Display operational matrix scale + st.markdown( + f"πŸ“Š **Analysis Profile Scope**: Mapped **{output['fg_count']}** significant foreground profiles out of **{output['bg_count']}** reference background items." + ) + + # Build multi-tab interface layer for ontology subcategories + tabs = st.tabs([ + "🧬 Biological Process (BP)", + "πŸ”¬ Cellular Component (CC)", + "πŸ§ͺ Molecular Function (MF)" + ]) + categories_data = output["categories"] + + for idx, go_type in enumerate(["BP", "CC", "MF"]): + with tabs[idx]: + fig = categories_data[go_type]["fig"] + df_go = categories_data[go_type]["df"] + + if fig is not None and df_go is not None: + # Render plotly bar figures generated straight from backend engine + st.plotly_chart(fig, use_container_width=True) + + st.subheader(f"πŸ“Š {go_type} Results Dataframe") + st.dataframe(df_go, use_container_width=True) + else: + st.info(f"No statistically overrepresented terms identified for Category: **{go_type}**") \ No newline at end of file diff --git a/content/filtering.py b/content/filtering.py new file mode 100644 index 0000000..2bad00c --- /dev/null +++ b/content/filtering.py @@ -0,0 +1,173 @@ +"""Filtering Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import filtering functions from openms_insight package +from openms_insight.analysis.filter import ( + filter_low_abundance, + filter_low_repeatability, + filter_low_variance, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Filtering") + +st.markdown( + """ +Filter out low-quality proteins from your dataset based on abundance, repeatability, or variance thresholds. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Identify actual sample columns dynamically +sample_cols = [ + c + for c in pivot_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Original Data View --- +st.subheader("Original Abundance Table") +st.markdown( + f"Currently displaying **{pivot_df.shape[0]}** proteins and **{len(sample_cols)}** samples before filtering." +) +st.dataframe(pivot_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Filter Configuration --- +st.subheader("Configure Filter Engine") + +# Prepare Polars Metadata DataFrame required by openms_insight functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for filtering strategy +filter_method = st.selectbox( + "Select Filtering Method", + options=["Low Abundance", "Low Repeatability", "Low Variance"], + index=0, + help="Choose the statistical criteria to prune unreliable protein entries.", +) + +# Render threshold sliders dynamically based on the selected filter method +if filter_method == "Low Abundance": + st.markdown( + "**Low Abundance Filter**: Keeps rows where at least one group's median is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +elif filter_method == "Low Repeatability": + st.markdown( + "**Low Repeatability Filter**: Keeps rows where at least one group has a missing value ratio within the allowed maximum." + ) + threshold = st.slider( + "Max Missing Ratio", + min_value=0.0, + max_value=100.0, + value=50.0, + step=5.0, + help="Allowed missing value (zero or null) ratio per group.", + ) + +elif filter_method == "Low Variance": + st.markdown( + "**Low Variance Filter**: Keeps rows where at least one group's variance is above the selected percentile threshold." + ) + threshold = st.slider( + "Threshold Percentile (%)", + min_value=0.0, + max_value=100.0, + value=10.0, + step=5.0, + ) + +# --- SECTION 3: Filter Execution and Collected Results View --- +if st.button("Apply Filter", type="primary"): + # Convert the original Pandas DataFrame into a Polars LazyFrame graph + quant_lazy = pl.from_pandas(pivot_df).lazy() + + # Route execution to the chosen openms_insight engine function + if filter_method == "Low Abundance": + filtered_lazy = filter_low_abundance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + elif filter_method == "Low Repeatability": + # Convert percent slider input to ratio expected by the function (e.g., 50.0% -> 0.5) + filtered_lazy = filter_low_repeatability( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + max_missing_ratio=threshold / 100.0, + ) + elif filter_method == "Low Variance": + filtered_lazy = filter_low_variance( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + threshold_percentile=threshold, + ) + + # Collect the evaluated lazy graph and convert back to Pandas for visualization + filtered_df = strip_stat_columns(filtered_lazy.collect().to_pandas()) + st.session_state["filtered_df"] = filtered_df + + # Layout response metrics and the filtered matrix + st.success(f"Successfully applied **{filter_method}** filter!") + + # Display dataset scale compression stats + col1, col2, col3 = st.columns(3) + col1.metric("Original Proteins", pivot_df.shape[0]) + col2.metric("Filtered Proteins", filtered_df.shape[0]) + col3.metric( + "Removed Proteins", pivot_df.shape[0] - filtered_df.shape[0], delta=None + ) + + st.subheader("Filtered Abundance Table") + if filtered_df.empty: + st.warning( + "The filtered table is empty. Try relaxing the threshold constraints." + ) + else: + st.dataframe(filtered_df, use_container_width=True) \ No newline at end of file diff --git a/content/imputation.py b/content/imputation.py new file mode 100644 index 0000000..4350263 --- /dev/null +++ b/content/imputation.py @@ -0,0 +1,145 @@ +"""Imputation Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map + +# Import imputation algorithms from openms_insight engine +from openms_insight.analysis.imputation import impute_mar, impute_smallest_value + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame) -> pd.DataFrame: + """Keep preprocessing tables intensity-only before statistical analysis.""" + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Missing Value Imputation") + +st.markdown( + """ +Handle missing values (zeros or nulls) in your quantification matrix using biological group-aware (MAR) or absolute lowest limit (MNAR) techniques. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load base dataset and clean dictionary keys +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# 1. Pipeline Checkpoint: Fetch upstream filtered data if available, fallback to raw pivot matrix +if "filtered_df" in st.session_state and st.session_state["filtered_df"] is not None: + base_df = strip_stat_columns(st.session_state["filtered_df"]) + st.session_state["filtered_df"] = base_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No filtering history found. Operating on the original unfiltered table." + ) + +# 2. Identify actual sample columns dynamically based on the current active matrix +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Input Matrix Summary --- +st.subheader("Input Matrix Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples before imputation." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Imputation Configuration --- +st.subheader("Configure Imputation Engine") + +# Build Polars structural metadata DataFrame +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +# User selection for core missingness assumption strategy +impute_category = st.selectbox( + "Select Imputation Class", + options=["MAR (Missing At Random)", "MNAR (Missing Not At Random)"], + index=0, + help="MAR uses group metrics (Mean/Median). MNAR shifts values below the limit of detection.", +) + +# Render algorithmic options sub-menus based on the parent selection +if impute_category == "MAR (Missing At Random)": + st.markdown( + "**Group Character Imputation**: Fills missing metrics leveraging sample properties belonging to the same group." + ) + strategy_opt = st.radio( + "Mathematical Strategy", + options=["median", "mean"], + index=0, + horizontal=True, + ) + +elif impute_category == "MNAR (Missing Not At Random)": + st.markdown( + "**Smallest Value Imputation**: Replaces missing items with the minimum values detected to reflect technical dropout limits." + ) + scope_opt = st.radio( + "Detection Minimum Scope", + options=["row", "global"], + index=0, + horizontal=True, + help="'row' targets current protein minimum; 'global' searches the entire mass spectrometry matrix profile.", + ) + +# --- SECTION 3: Imputation Execution --- +if st.button("Apply Imputation", type="primary"): + # Initialize optimization pipeline graph via lazy loading conversion + quant_lazy = pl.from_pandas(base_df).lazy() + + # Route configuration matrix parameters to designated engine function channels + if impute_category == "MAR (Missing At Random)": + imputed_lazy = impute_mar( + quantification_data=quant_lazy, + metadata=metadata_pl, + group_column="group", + strategy=strategy_opt, + ) + elif impute_category == "MNAR (Missing Not At Random)": + imputed_lazy = impute_smallest_value( + quantification_data=quant_lazy, metadata=metadata_pl, scope=scope_opt + ) + + # Resolve lazy graph optimization tree and push to display data frame structure + imputed_df = strip_stat_columns(imputed_lazy.collect().to_pandas()) + + # πŸ’Ύ Save current output into Session State for down-stream processing (Normalization, Statistics) + st.session_state["imputed_df"] = imputed_df + + st.success(f"Successfully finalized **{impute_category}** imputation step!") + + # Calculate and display a quick performance matrix check + st.subheader("Imputed Result Table") + st.dataframe(imputed_df, use_container_width=True) \ No newline at end of file diff --git a/content/normalization.py b/content/normalization.py new file mode 100644 index 0000000..c0e97e8 --- /dev/null +++ b/content/normalization.py @@ -0,0 +1,242 @@ +"""Normalization Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import normalization engine functions from openms_insight +from openms_insight.analysis.normalization import ( + normalize_samples, + scale_data, + transform_data, +) + +STAT_COLUMNS = ["log2FC", "p-value", "p-adj", "stat"] + + +def strip_stat_columns(df: pd.DataFrame | None) -> pd.DataFrame | None: + """Keep preprocessing tables intensity-only before statistical analysis.""" + if df is None: + return None + return df.drop(columns=[c for c in STAT_COLUMNS if c in df.columns], errors="ignore") + +params = page_setup() +st.title("Data Normalization & Scaling") + +st.markdown( + """ +Standardize and transform your protein abundance profiles to correct for technical variations and optimize statistical distributions. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +pivot_df = strip_stat_columns(pivot_df) +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +filtered_df = strip_stat_columns(st.session_state.get("filtered_df")) +imputed_df = strip_stat_columns(st.session_state.get("imputed_df")) +normalized_df = strip_stat_columns(st.session_state.get("normalized_df")) +if filtered_df is not None: + st.session_state["filtered_df"] = filtered_df +if imputed_df is not None: + st.session_state["imputed_df"] = imputed_df +if normalized_df is not None: + st.session_state["normalized_df"] = normalized_df + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = imputed_df + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = filtered_df + st.warning( + "⚠️ **Imputation Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original unfiltered table." + ) + +# 2. Extract actual active sample columns dynamically +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently displaying **{base_df.shape[0]}** rows and **{len(sample_cols)}** samples entering the normalization block." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("### Pipeline Overview") +st.caption("Data flows in order: Filtering -> Imputation -> Normalization") + +step_rows = [ + { + "Step": "Filtering", + "Status": "Done" if filtered_df is not None else "Not run", + "Rows": filtered_df.shape[0] if filtered_df is not None else "-", + "Cols": filtered_df.shape[1] if filtered_df is not None else "-", + }, + { + "Step": "Imputation", + "Status": "Done" if imputed_df is not None else "Not run", + "Rows": imputed_df.shape[0] if imputed_df is not None else "-", + "Cols": imputed_df.shape[1] if imputed_df is not None else "-", + }, + { + "Step": "Normalization", + "Status": "Done" if normalized_df is not None else "Not run", + "Rows": normalized_df.shape[0] if normalized_df is not None else "-", + "Cols": normalized_df.shape[1] if normalized_df is not None else "-", + }, +] +st.dataframe(pd.DataFrame(step_rows), hide_index=True, use_container_width=True) + +with st.expander("Show step tables", expanded=False): + if filtered_df is not None: + st.markdown("#### Filtering output") + st.dataframe(filtered_df.head(10), use_container_width=True) + if imputed_df is not None: + st.markdown("#### Imputation output") + st.dataframe(imputed_df.head(10), use_container_width=True) + if normalized_df is not None: + st.markdown("#### Normalization output") + st.dataframe(normalized_df.head(10), use_container_width=True) + if filtered_df is None and imputed_df is None and normalized_df is None: + st.info("No preprocessing outputs yet. Start from Filtering.") + +st.markdown("---") + +# --- SECTION 2: Normalization Parameter Configuration --- +st.subheader("Configure Preprocessing & Scaling Chains") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2, col3 = st.columns(3) + +with col1: + st.markdown("### 🧬 1. Mathematical Transformation") + transform_strategy = st.selectbox( + "Select Transformation", + options=["None", "log2", "log10", "square_root", "cube_root"], + index=0, + help="Compress data dynamic range and stabilize heteroscedastic variance profiles.", + ) + +with col2: + st.markdown("### πŸ§ͺ 2. Sample Normalization") + norm_strategy = st.selectbox( + "Select Normalization", + options=["None", "sum", "median", "pqn", "reference_feature", "quantile"], + index=0, + help="Perform column-wise corrections to account for variable sample loading concentrations.", + ) + + # Conditionally display target input field for reference feature matching + ref_feature_input = None + if norm_strategy == "reference_feature": + ref_feature_input = st.text_input( + "Reference Protein Name (ID)", + value="", + placeholder="e.g., P01234 or GAPDH", + help=f"Enter the exact unique identifier string matching a key inside the '{id_col}' column.", + ) + +with col3: + st.markdown("### πŸ“Š 3. Row Scaling") + scaling_strategy = st.selectbox( + "Select Scaling Mode", + options=["None", "mean_centering", "auto_scaling", "pareto_scaling", "range_scaling"], + index=0, + help="Adjust individual feature weights to make low and high abundance proteins comparable.", + ) + + +# --- SECTION 3: Normalization Pipe Sequential Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Apply Normalization Pipelines", type="primary"): + + # Validate reference feature selection if active before hitting polars execution layers + if norm_strategy == "reference_feature" and not ref_feature_input: + st.error( + "❌ Validation Error: Please provide a valid Reference Protein Name to use the 'reference_feature' strategy." + ) + st.stop() + + # Convert pandas memory buffer into optimization lazy dataframe tree graph + processing_lazy = pl.from_pandas(base_df).lazy() + + # Execute Chain 1: Transform Matrix Data + try: + processing_lazy = transform_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=transform_strategy, + ) + + # Execute Chain 2: Normalize Sample Intensities (Columns) + processing_lazy = normalize_samples( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=norm_strategy, + id_col=id_col, + reference_feature=ref_feature_input if norm_strategy == "reference_feature" else None, + ) + + # Execute Chain 3: Scale Individual Features (Rows) + processing_lazy = scale_data( + quantification_data=processing_lazy, + metadata=metadata_pl, + strategy=scaling_strategy, + ) + + # Finalize and collect pipeline query graph optimizations + normalized_df = strip_stat_columns(processing_lazy.collect().to_pandas()) + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (Statistics Block) + st.session_state["normalized_df"] = normalized_df + + st.success("Successfully executed all selected normalization pipelines!") + + # Display the finalized transformation matrix view + st.subheader("Normalized Abundance Table") + st.dataframe(normalized_df, use_container_width=True) + + except ValueError as val_err: + # Gracefully handle validation failures raised from the engine layers (e.g., missing reference protein) + st.error(f"Engine Configuration Error: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/content/results_abundance.py b/content/results_abundance.py index a7ff453..38c42bc 100644 --- a/content/results_abundance.py +++ b/content/results_abundance.py @@ -1,9 +1,11 @@ """Abundance (ProteomicsLFQ) Results Page.""" import streamlit as st import pandas as pd +import numpy as np from pathlib import Path from src.common.common import page_setup from src.common.results_helpers import get_workflow_dir, get_abundance_data +from src.workflow.ParameterManager import ParameterManager params = page_setup() st.title("Abundance Quantification") @@ -11,7 +13,7 @@ st.markdown( """ View protein and PSM-level quantification from **ProteomicsLFQ**. -This page calculates differential expression statistics between sample groups. +This page focuses on raw abundance intensity for preprocessing. """ ) @@ -21,6 +23,10 @@ workflow_dir = get_workflow_dir(st.session_state["workspace"]) quant_dir = workflow_dir / "results" / "quant_results" +parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") + +workflow_params = parameter_manager.get_parameters_from_json() +analysis_mode = workflow_params.get("analysis-mode", "LFQ") if not quant_dir.exists(): st.info("No quantification results available yet. Please run the workflow first.") @@ -35,6 +41,55 @@ csv_file = csv_files[0] +def render_protein_table(pivot_df, is_lfq=True): + """Common function to render the protein-level abundance table""" + pivot_df = pivot_df.copy() + st.markdown("### Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples." + ) + + if is_lfq: + # Handle LFQ mode columns (Raw Intensity) + id_col = "ProteinName" + exclude_cols = [id_col, "PeptideSequence"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + display_cols = [id_col, "Intensity"] + sample_cols + ["PeptideSequence"] + help_text = "Raw sample intensities" + y_min = None + else: + # Handle non-LFQ mode columns (Log2-transformed Intensity) + id_col = "protein" + exclude_cols = [id_col, "n_proteins", "n_peptides", "protein_score"] + sample_cols = [c for c in pivot_df.columns if c not in exclude_cols and "ratio" not in c.lower()] + + pivot_df["Intensity"] = pivot_df[sample_cols].apply( + lambda row: [np.log2(v + 1) for v in row], axis=1 + ) + display_cols = [id_col, "Intensity"] + sample_cols + help_text = "Sample intensities (log2 scale)" + y_min = 0 + + # Filter to available columns, then sort and display + available_cols = [c for c in display_cols if c in pivot_df.columns] + view_df = pivot_df[available_cols] + + st.dataframe( + view_df, + column_config={ + "Intensity": st.column_config.BarChartColumn( + "Intensity", + help=help_text, + width="small", + y_min=y_min, + ), + }, + use_container_width=True, + ) + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) try: @@ -44,68 +99,53 @@ st.info("No data found in this file.") st.stop() - with protein_tab: - st.markdown("### Protein-Level Abundance Table") + result = get_abundance_data(st.session_state["workspace"]) - st.info( - "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." - ) + if analysis_mode == "LFQ": + protein_tab, psm_tab = st.tabs(["Protein Table", "PSM-level Quantification Table"]) - result = get_abundance_data(st.session_state["workspace"]) - if result is None: - st.warning("Could not compute abundance data. Please ensure sample groups are defined in the Configure page.") - st.page_link("content/workflow_configure.py", label="Go to Configure", icon="βš™οΈ") - st.stop() + with protein_tab: + if result is None: + st.warning("Could not load abundance data. Please run the workflow first.") + st.stop() + + pivot_df, expr_df, group_map = result + render_protein_table(pivot_df, is_lfq=True) - pivot_df, expr_df, group_map = result + with psm_tab: + st.markdown("### PSM-level Quantification Table") + st.info( + "This table shows the PSM-level quantification data, including protein IDs, " + "peptide sequences, charge states, and intensities across samples. " + "Each row represents one peptide-spectrum match detected from the MS/MS analysis." + ) + st.dataframe(df, use_container_width=True) - # Display group comparison info - groups = sorted(set(group_map.values())) - if len(groups) >= 2: - group1, group2 = sorted(groups)[:2] - st.info(f"Statistical comparison: **{group2} vs {group1}**") + else: + pre_processing_tab, protein_tab = st.tabs(["Pre-processing", "Protein Table"]) - # Get sample columns (between stats and PeptideSequence) - sample_cols = [c for c in pivot_df.columns if c not in ["ProteinName", "log2FC", "p-value", "PeptideSequence"]] + if result is None: + st.info("πŸ’‘ Please run the workflow first to see results.") + st.stop() - pivot_df["Intensity"] = pivot_df[sample_cols].apply(list, axis=1) + pivot_df, expr_df, group_map = result - # Reorder columns: place Intensity after p-value - display_cols = ["ProteinName", "log2FC", "p-value", "Intensity"] + sample_cols + ["PeptideSequence"] - display_df = pivot_df[display_cols] - - st.dataframe( - display_df.sort_values("p-value"), - column_config={ - "Intensity": st.column_config.BarChartColumn( - "Intensity", - help="Raw sample intensities", - width="small", - ), - }, - use_container_width=True, - ) + with pre_processing_tab: + st.write("### Final Results (Intensity matrix)") + st.dataframe(pivot_df.head(10)) - with psm_tab: - st.markdown("### PSM-level Quantification Table") - st.info( - "This table shows the PSM-level quantification data, including protein IDs, " - "peptide sequences, charge states, and intensities across samples. " - "Each row represents one peptide-spectrum match detected from the MS/MS analysis." - ) - st.dataframe(df, use_container_width=True) + with protein_tab: + render_protein_table(pivot_df, is_lfq=False) except Exception as e: st.error(f"Failed to load {csv_file.name}: {e}") st.markdown("---") -st.markdown("**Next steps:** Explore statistical visualizations") +st.markdown("**Next steps:** Continue preprocessing, then run statistical inference") col1, col2, col3 = st.columns(3) with col1: - st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") + st.page_link("content/filtering.py", label="Filtering", icon="🧹") with col2: - st.page_link("content/results_pca.py", label="PCA", icon="πŸ“Š") + st.page_link("content/imputation.py", label="Imputation", icon="🧩") with col3: - st.page_link("content/results_heatmap.py", label="Heatmap", icon="πŸ”₯") + st.page_link("content/statistical.py", label="Statistical Inference", icon="πŸ”¬") \ No newline at end of file diff --git a/content/results_heatmap.py b/content/results_heatmap.py index 4ece3f4..104bff6 100644 --- a/content/results_heatmap.py +++ b/content/results_heatmap.py @@ -1,19 +1,18 @@ """Heatmap Results Page.""" import streamlit as st import numpy as np -import plotly.express as px -from scipy.cluster.hierarchy import linkage, leaves_list -from scipy.spatial.distance import pdist +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import Heatmap params = page_setup() st.title("Heatmap") st.markdown( """ -Hierarchically clustered heatmap of protein-level abundance (Z-score normalized). -Proteins and samples are ordered by similarity. +Interactive hierarchically clustered heatmap of protein-level abundance (Z-score normalized). +Powered by OpenMS-Insight multi-resolution engine. """ ) @@ -28,42 +27,68 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) -top_n = st.slider("Number of proteins", 20, 200, 50, key="heatmap_top_n") +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +sample_cols = expr_df.columns.tolist() +# UI settings (number of top variance proteins) +top_n = st.slider("Number of proteins (Highest Variance)", 20, 200, 50, key="heatmap_top_n") + +# Process data (variance selection -> Z-score normalization) var_series = expr_df.var(axis=1) top_proteins = var_series.sort_values(ascending=False).head(top_n).index heatmap_df = expr_df.loc[top_proteins] + +# Compute Z-scores and clean missing/invalid values heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() if not heatmap_z.empty: - row_linkage = linkage(pdist(heatmap_z.values), method="average") - row_order = leaves_list(row_linkage) + # Melt and convert data to Polars to satisfy OpenMS-Insight component requirements + # Restore the id column from the index as a regular column + heatmap_z_reset = heatmap_z.reset_index() - col_linkage = linkage(pdist(heatmap_z.T.values), method="average") - col_order = leaves_list(col_linkage) + # Unpivot the wide-format matrix into long-format (X, Y, Intensity) + melted_df = heatmap_z_reset.melt( + id_vars=[id_col], + value_vars=sample_cols, + var_name="Sample", + value_name="Z_score" + ) - heatmap_clustered = heatmap_z.iloc[row_order, col_order] + # Add sample group mapping if available for heatmap categories + if sample_group_map: + melted_df["Group"] = melted_df["Sample"].map(sample_group_map) - fig_heatmap = px.imshow( - heatmap_clustered, - labels=dict(x="Sample", y="Protein", color="Z-score"), - aspect="auto", - color_continuous_scale=[[0.0, "#3b6fb6"], [0.5, "white"], [1.0, "#b40426"]], - zmin=-3, zmax=3 - ) + # Pack the Pandas DataFrame into a Polars LazyFrame + heatmap_pl_lazy = pl.from_pandas(melted_df).lazy() - fig_heatmap.update_layout( - height=700, - xaxis={'side': 'bottom'}, - yaxis={'side': 'left'} + # Initialize the OpenMS-Insight Heatmap component and map attributes + heatmap_component = Heatmap( + cache_id="quantms_protein_heatmap", + x_column="Sample", + y_column=id_col, + data=heatmap_pl_lazy, + intensity_column="Z_score", + title="Protein Abundance Heatmap (Z-score)", + x_label="Samples", + y_label="Proteins", + colorscale="RdBu", + reversescale=True, + log_scale=False, # Z-score can be negative, so log scale must stay off + intensity_label="Z-score", + category_column=None, + min_points=10000, # Generous point-count ceiling so the full grid renders ) - fig_heatmap.update_xaxes(tickfont=dict(size=10)) - fig_heatmap.update_yaxes(tickfont=dict(size=8)) - - st.plotly_chart(fig_heatmap, use_container_width=True) + # Render the component + state_manager = st.session_state.get("state") + heatmap_component(state_manager=state_manager) else: st.warning("Insufficient data to generate the heatmap.") diff --git a/content/results_heatmap_clustered.py b/content/results_heatmap_clustered.py new file mode 100644 index 0000000..7104c3a --- /dev/null +++ b/content/results_heatmap_clustered.py @@ -0,0 +1,107 @@ +"""Clustered Heatmap Results Page.""" +import streamlit as st +import numpy as np +import polars as pl +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import ClusteredHeatmap + +params = page_setup() +st.title("Clustered Heatmap") + +st.markdown( + """ +A real grid heatmap (rows = proteins, columns = samples) with hierarchical +clustering dendrograms on both axes and a sample-group color bar, powered +by OpenMS-Insight. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +if expr_df.empty: + st.info("No data available for heatmap.") + st.stop() + +top_n = st.slider("Number of proteins (Highest Variance)", 10, 200, 30, key="clustered_heatmap_top_n") + +var_series = expr_df.var(axis=1) +top_proteins = var_series.sort_values(ascending=False).head(top_n).index +heatmap_df = expr_df.loc[top_proteins] + +heatmap_z = heatmap_df.sub(heatmap_df.mean(axis=1), axis=0).div(heatmap_df.std(axis=1), axis=0) +heatmap_z = heatmap_z.replace([np.inf, -np.inf], np.nan).dropna() + +if heatmap_z.empty: + st.warning("Insufficient data to generate the heatmap.") + st.stop() + +heatmap_z_reset = heatmap_z.reset_index() +heatmap_lazy = pl.from_pandas(heatmap_z_reset).lazy() + +sample_cols = expr_df.columns.tolist() +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, +) + +# Assign group annotation-bar colors in sorted-group order (matching how +# ClusteredHeatmap._preprocess() orders unique groups internally). +group_palette = [ + "#00BFC4", # teal + "#F8766D", # salmon + "#7CAE00", # yellow-green + "#C77CFF", # lavender purple + "#E7B800", # gold/amber + "#619CFF", # blue + "#FF61C3", # pink/magenta + "#00BA38", # green + "#FF8C42", # orange + "#00B0F6", # sky blue +] +unique_groups = sorted(set(sample_group_map.values())) +group_colors = {g: group_palette[i % len(group_palette)] for i, g in enumerate(unique_groups)} + +heatmap_component = ClusteredHeatmap( + cache_id="quantms_clustered_heatmap", + cache_path=str(st.session_state["workspace"]), + id_col=id_col, + data=heatmap_lazy, + metadata=metadata_pl, + row_cluster=True, + col_cluster=True, + title="Protein Abundance Heatmap (Z-score, clustered)", + x_label="Samples", + y_label="Proteins", + colorscale=[[0, "#6699E0"], [0.5, "#FFFFFF"], [1, "#E06666"]], + reversescale=False, + intensity_label="Z-score", + group_colors=group_colors, +) + +state_manager = st.session_state.get("state") +# Scale height with the number of proteins so row labels stay readable - +# BaseComponent otherwise defaults to a flat 400px, too short for a +# dendrogram+heatmap composite with more than a handful of rows. +heatmap_height = max(600, min(1400, 300 + top_n * 20)) +heatmap_component(state_manager=state_manager, height=heatmap_height) + +st.markdown("---") +st.markdown("**Other visualizations:**") +col1, col2 = st.columns(2) +with col1: + st.page_link("content/results_volcano.py", label="Volcano Plot", icon="πŸŒ‹") +with col2: + st.page_link("content/results_heatmap.py", label="Heatmap (original)", icon="πŸ”₯") diff --git a/content/results_pathway_analysis.py b/content/results_pathway_analysis.py new file mode 100644 index 0000000..f5eb4c1 --- /dev/null +++ b/content/results_pathway_analysis.py @@ -0,0 +1,258 @@ +import json +import mygene +import streamlit as st +import pandas as pd +import numpy as np +import plotly.express as px +import plotly.io as pio +from collections import defaultdict +from scipy.stats import fisher_exact +from pathlib import Path +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data + +# ================================ +# Page setup +# ================================ +params = page_setup() +st.title("ProteomicsLFQ Results") + +# ================================ +# Workspace check +# ================================ +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# ================================ +# _run_go_enrichment function +# ================================ +def _run_go_enrichment(pivot_df: pd.DataFrame, results_dir: Path): + p_cutoff = 0.05 + fc_cutoff = 1.0 + + analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() + + if analysis_df.empty: + st.error("No valid statistical data found for GO enrichment.") + st.write("❗ analysis_df is empty") + else: + with st.spinner("Fetching GO terms from MyGene.info API..."): + mg = mygene.MyGeneInfo() + + def get_clean_uniprot(name): + parts = str(name).split("|") + return parts[1] if len(parts) >= 2 else parts[0] + + analysis_df["UniProt"] = analysis_df["protein"].apply(get_clean_uniprot) + + bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() + fg_ids = analysis_df[ + (analysis_df["p-value"] < p_cutoff) & + (analysis_df["log2FC"].abs() >= fc_cutoff) + ]["UniProt"].dropna().astype(str).unique().tolist() + # st.write("βœ… get_clean_uniprot applied") + + if len(fg_ids) < 3: + st.warning( + f"Not enough significant proteins " + f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " + f"Found: {len(fg_ids)}" + ) + st.write("❗ Not enough significant proteins") + else: + res_list = mg.querymany( + bg_ids, scopes="uniprot", fields="go", as_dataframe=False + ) + res_go = pd.DataFrame(res_list) + if "notfound" in res_go.columns: + res_go = res_go[res_go["notfound"] != True] + + def extract_go_terms(go_data, go_type): + if not isinstance(go_data, dict) or go_type not in go_data: + return [] + terms = go_data[go_type] + if isinstance(terms, dict): + terms = [terms] + return list({t.get("term") for t in terms if "term" in t}) + + for go_type in ["BP", "CC", "MF"]: + res_go[f"{go_type}_terms"] = res_go["go"].apply( + lambda x: extract_go_terms(x, go_type) + ) + + annotated_ids = set(res_go["query"].astype(str)) + fg_set = annotated_ids.intersection(fg_ids) + bg_set = annotated_ids + # st.write(f"βœ… fg_set bg_set are set") + + def run_go(go_type): + go2fg = defaultdict(set) + go2bg = defaultdict(set) + + for _, row in res_go.iterrows(): + uid = str(row["query"]) + for term in row[f"{go_type}_terms"]: + go2bg[term].add(uid) + if uid in fg_set: + go2fg[term].add(uid) + + records = [] + N_fg = len(fg_set) + N_bg = len(bg_set) + + for term, fg_genes in go2fg.items(): + a = len(fg_genes) + if a == 0: + continue + b = N_fg - a + c = len(go2bg[term]) - a + d = N_bg - (a + b + c) + + _, p = fisher_exact([[a, b], [c, d]], alternative="greater") + records.append({ + "GO_Term": term, + "Count": a, + "GeneRatio": f"{a}/{N_fg}", + "p_value": p, + }) + + df = pd.DataFrame(records) + if df.empty: + return None, None + + df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) + df = df.sort_values("p_value").head(20) + + # βœ… Plotly Figure + fig = px.bar( + df, + x="-log10(p)", + y="GO_Term", + orientation="h", + title=f"GO Enrichment ({go_type})", + ) + + # st.write(f"βœ… Plotly Figure generated") + + fig.update_layout( + yaxis=dict(autorange="reversed"), + height=500, + margin=dict(l=10, r=10, t=40, b=10), + ) + + return fig, df + + go_results = {} + + for go_type in ["BP", "CC", "MF"]: + fig, df_go = run_go(go_type) + if fig is not None: + go_results[go_type] = { + "fig": fig, + "df": df_go + } + # st.write(f"βœ… go_type generated") + + go_dir = results_dir / "go-terms" + go_dir.mkdir(parents=True, exist_ok=True) + + go_data = {} + + for go_type in ["BP", "CC", "MF"]: + if go_type in go_results: + fig = go_results[go_type]["fig"] + df = go_results[go_type]["df"] + + go_data[go_type] = { + "fig_json": fig.to_json(), # Figure β†’ JSON string + "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts + } + + go_json_file = go_dir / "go_results.json" + with open(go_json_file, "w") as f: + json.dump(go_data, f) + st.session_state["go_results"] = go_results + st.session_state["go_ready"] = True if go_data else False + # st.write("βœ… GO enrichment analysis complete") + +# ================================ +# Load abundance data +# ================================ +results_dir = Path(st.session_state["workspace"]) / "topp-workflow" / "results" / "quant_results" +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info("Abundance data not available. Please run the workflow and configure sample groups first.") + st.page_link("content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹") + st.stop() + +pivot_df, expr_df, group_map = result + +go_json_file = results_dir / "go-terms" / "go_results.json" + +go_input_df = pivot_df.copy() +if "ProteinName" in go_input_df.columns: + go_input_df = go_input_df.rename(columns={"ProteinName": "protein"}) + +_run_go_enrichment(go_input_df, results_dir) + +# ================================ +# Tabs +# ================================ +protein_tab, = st.tabs(["🧬 Protein Table"]) + +# ================================ +# Protein-level results +# ================================ +with protein_tab: + st.markdown("### 🧬 Protein-Level Abundance Table") + st.info( + "This protein-level table is generated by grouping all PSMs that map to the " + "same protein and aggregating their intensities across samples.\n\n" + "Additionally, log2 fold change and p-values are calculated between sample groups." + ) + + if pivot_df.empty: + st.info("No protein-level data available.") + else: + st.session_state["pivot_df"] = pivot_df + st.dataframe(pivot_df.sort_values("p-value"), width="stretch") + +# ====================================================== +# GO Enrichment Results +# ====================================================== +st.markdown("---") +st.subheader("🧬 GO Enrichment Analysis") + +if not go_json_file.exists(): + st.info("GO Enrichment results are not available yet. Please run the analysis first.") +else: + with open(go_json_file, "r") as f: + go_data = json.load(f) + + bp_tab, cc_tab, mf_tab = st.tabs([ + "🧬 Biological Process", + "🏠 Cellular Component", + "βš™οΈ Molecular Function", + ]) + + for tab, go_type in zip([bp_tab, cc_tab, mf_tab], ["BP", "CC", "MF"]): + with tab: + if go_type not in go_data: + st.info(f"No enriched {go_type} terms found.") + continue + + fig_json = go_data[go_type]["fig_json"] + df_dict = go_data[go_type]["df_dict"] + + fig = pio.from_json(fig_json) + + df_go = pd.DataFrame(df_dict) + + if df_go.empty: + st.info(f"No enriched {go_type} terms found.") + else: + st.plotly_chart(fig, width="stretch") + + st.markdown(f"#### {go_type} Enrichment Results") + st.dataframe(df_go, width="stretch") \ No newline at end of file diff --git a/content/results_pca.py b/content/results_pca.py index 45ea8eb..29f441a 100644 --- a/content/results_pca.py +++ b/content/results_pca.py @@ -1,11 +1,10 @@ """PCA Results Page.""" -import streamlit as st import pandas as pd -import plotly.express as px -from sklearn.decomposition import PCA -from sklearn.preprocessing import StandardScaler +import polars as pl +import streamlit as st from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +from openms_insight import PCAPlot params = page_setup() st.title("PCA Analysis") @@ -13,7 +12,7 @@ st.markdown( """ Principal Component Analysis (PCA) of protein-level abundance. -Samples are colored by group assignment to visualize clustering. +Samples are projected onto their principal components and colored by group assignment to visualize clustering. """ ) @@ -21,6 +20,7 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Load abundance data (base wide-format table + sample -> group mapping) result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -28,60 +28,143 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +# Mirrors statistical.py: PCA should run on the most-processed data available. +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns + if c not in [id_col, "PeptideSequence", "log2FC", "p-adj", "stat", "p-value"] +] +unique_groups = sorted({sample_group_map[s] for s in sample_cols if s in sample_group_map}) + +if len(sample_cols) < 2: + st.info("PCA requires at least 2 samples.") + st.stop() -top_n = 500 +if len(unique_groups) < 2: + st.warning( + "Only one biological group was detected - points will still be plotted, " + "but group-based coloring requires 2 or more groups." + ) -top_proteins = ( - pivot_df - .dropna(subset=["p-adj"]) - .sort_values("p-adj", ascending=True) - .head(top_n)["ProteinName"] +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples " + f"belonging to **{len(unique_groups)} groups** ({', '.join(unique_groups)})." ) +st.dataframe(base_df, use_container_width=True) -expr_df_pca = expr_df.loc[ - expr_df.index.intersection(top_proteins) -] +st.markdown("---") + +# --- SECTION 2: PCA Configuration --- +st.subheader("Configure PCA") + +expr_df_wide = base_df.set_index(id_col)[sample_cols] +max_available = expr_df_wide.shape[0] + +if max_available <= 20: + top_n = max_available + st.caption(f"Using all {top_n} proteins for PCA (dataset too small for variance filtering).") +else: + top_n = st.slider( + "Number of proteins (Highest Variance)", + min_value=20, + max_value=min(5000, max_available), + value=min(500, max_available), + step=10, + key="pca_top_n", + help=( + "PCA is computed only on the N proteins with the highest variance " + "across samples, to reduce noise from low-variance/uninformative features." + ), + ) + +top_proteins = expr_df_wide.var(axis=1).sort_values(ascending=False).head(top_n).index +expr_df_pca = expr_df_wide.loc[top_proteins].reset_index() if expr_df_pca.shape[0] < 2: - st.info("Not enough proteins after p-value filtering for PCA.") + st.info("Not enough proteins after variance filtering for PCA.") st.stop() -X = expr_df_pca.T -X_scaled = StandardScaler().fit_transform(X) - -pca = PCA(n_components=2) -pcs = pca.fit_transform(X_scaled) - -pca_df = pd.DataFrame( - pcs, - columns=["PC1", "PC2"], - index=X.index +# Prepare structural Polars metadata DataFrame required by PCAPlot +metadata_pl = pl.DataFrame( + [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map], + schema={"sample_id": pl.String, "group": pl.String}, ) +pca_lazy = pl.from_pandas(expr_df_pca).lazy() + +# 3. Initialize the OpenMS-Insight PCAPlot component (computes PCA internally) +try: + pca_component = PCAPlot( + cache_id="quantms_pca_plot", + data=pca_lazy, + metadata=metadata_pl, + sample_id_field="sample_id", + group_field="group", + n_components=5, + title="Sample PCA", + ) +except ValueError as e: + st.error(f"PCA computation failed: {e}") + st.stop() -norm_map = { - k.replace(".mzML", ""): v - for k, v in group_map.items() -} -pca_df["Group"] = pca_df.index.map(norm_map) - -fig_pca = px.scatter( - pca_df, - x="PC1", - y="PC2", - color="Group", - text=pca_df.index, -) +variance_ratio = pca_component.get_variance_ratio() +pc_columns = pca_component.get_pc_columns() -fig_pca.update_traces(textposition="top center") -fig_pca.update_layout( - xaxis_title=f"PC1 ({pca.explained_variance_ratio_[0]*100:.1f}%)", - yaxis_title=f"PC2 ({pca.explained_variance_ratio_[1]*100:.1f}%)", - height=600, -) +# 4. Let the user pick which component pair to view (no recomputation needed) +col1, col2 = st.columns(2) +with col1: + pc_x_label = st.selectbox("X-axis component", pc_columns, index=0, key="pca_pc_x") +with col2: + default_y_index = 1 if len(pc_columns) > 1 else 0 + pc_y_label = st.selectbox("Y-axis component", pc_columns, index=default_y_index, key="pca_pc_y") + +pc_x = int(pc_x_label.replace("PC", "")) +pc_y = int(pc_y_label.replace("PC", "")) -st.plotly_chart(fig_pca, use_container_width=True) +# 5. Render the component +state_manager = st.session_state.get("state") +pca_component(state_manager=state_manager, pc_x=pc_x, pc_y=pc_y, height=600) -st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by p-adj)") +st.markdown( + "**Explained variance:** " + + ", ".join(f"{col} {ratio * 100:.1f}%" for col, ratio in zip(pc_columns, variance_ratio)) +) +st.markdown(f"**Proteins used:** {expr_df_pca.shape[0]} (top {top_n} by variance)") st.markdown("---") st.markdown("**Other visualizations:**") diff --git a/content/results_proteomicslfq.py b/content/results_proteomicslfq.py index 77eb332..fde2ab9 100644 --- a/content/results_proteomicslfq.py +++ b/content/results_proteomicslfq.py @@ -45,15 +45,14 @@ st.markdown("### 🧬 Protein-Level Abundance Table") st.info( "This protein-level table is generated by grouping all PSMs that map to the " - "same protein and aggregating their intensities across samples.\n\n" - "Additionally, log2 fold change and p-values are calculated between sample groups." + "same protein and aggregating their intensities across samples." ) if pivot_df.empty: st.info("No protein-level data available.") else: st.session_state["pivot_df"] = pivot_df - st.dataframe(pivot_df.sort_values("p-value"), use_container_width=True) + st.dataframe(pivot_df, use_container_width=True) # ====================================================== # GO Enrichment Results diff --git a/content/results_volcano.py b/content/results_volcano.py index 8502489..db2702f 100644 --- a/content/results_volcano.py +++ b/content/results_volcano.py @@ -1,9 +1,9 @@ """Volcano Plot Results Page.""" import streamlit as st -import plotly.express as px -import numpy as np +import polars as pl from src.common.common import page_setup -from src.common.results_helpers import get_abundance_data +from src.common.results_helpers import get_abundance_data, get_id_column +from openms_insight import VolcanoPlot params = page_setup() st.title("Volcano Plot") @@ -19,6 +19,19 @@ st.warning("Please initialize your workspace first.") st.stop() +# 1. Check if statistical analysis results are available in the session state +if "statistics_df" not in st.session_state or st.session_state["statistics_df"] is None: + st.info("Statistical analysis data not found. Please run the statistical engine first.") + st.page_link("content/statistical.py", label="Go to Statistical Inference", icon="πŸ”¬") + st.stop() + +# Retrieve the completed statistical analysis DataFrame +statistics_df = st.session_state["statistics_df"] + +if statistics_df.empty: + st.info("No data available for volcano plot.") + st.stop() + result = get_abundance_data(st.session_state["workspace"]) if result is None: st.info("Abundance data not available. Please run the workflow and configure sample groups first.") @@ -26,16 +39,13 @@ st.stop() pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) -if pivot_df.empty: - st.info("No data available for volcano plot.") - st.stop() - -volcano_df = pivot_df.copy() -volcano_df = volcano_df.dropna(subset=["log2FC", "p-adj"]) - -volcano_df["neg_log10_padj"] = -np.log10(volcano_df["p-adj"]) +# 2. Clean data and convert to Polars for component input +volcano_df = statistics_df.dropna(subset=["log2FC", "p-adj"]).copy() +volcano_pl_lazy = pl.from_pandas(volcano_df).lazy() +# 3. Configure UI sliders (changing thresholds does not invalidate cache) fc_thresh = st.slider( "log2 Fold Change threshold", min_value=0.5, @@ -52,49 +62,34 @@ step=0.001, ) -volcano_df["Significance"] = "Not significant" -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh), - "Significance", -] = "Up-regulated" - -volcano_df.loc[ - (volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh), - "Significance", -] = "Down-regulated" - -fig_volcano = px.scatter( - volcano_df, - x="log2FC", - y="neg_log10_padj", - color="Significance", - hover_data=["ProteinName", "log2FC", "p-value", "p-adj"], - color_discrete_map={ - "Up-regulated": "red", - "Down-regulated": "blue", - "Not significant": "lightgrey", - } +# 4. Initialize the OpenMS-Insight VolcanoPlot component +volcano_plot_component = VolcanoPlot( + cache_id="quantms_volcano_plot", + data=volcano_pl_lazy, + log2fc_column="log2FC", + pvalue_column="p-adj", + label_column=id_col, + up_color="#E74C3C", + down_color="#3498DB", + ns_color="#95A5A6", + show_threshold_lines=True, + threshold_line_style="dash", ) -fig_volcano.add_vline(x=fc_thresh, line_dash="dash") -fig_volcano.add_vline(x=-fc_thresh, line_dash="dash") -fig_volcano.add_hline(y=-np.log10(p_thresh), line_dash="dash") - -# Make x-axis symmetric around zero -max_abs_fc = volcano_df["log2FC"].abs().max() -x_range = [-max_abs_fc * 1.1, max_abs_fc * 1.1] # 10% padding +# 5. Render the component +state_manager = st.session_state.get("state") # Inject the project state management object -fig_volcano.update_layout( - xaxis_title="log2 Fold Change", - yaxis_title="-log10(p-adj)", - xaxis_range=x_range, +volcano_plot_component( + state_manager=state_manager, + fc_threshold=fc_thresh, + p_threshold=p_thresh, + max_labels=10, # Display labels for the top N significant proteins height=600, ) -st.plotly_chart(fig_volcano, use_container_width=True) - -up_count = (volcano_df["Significance"] == "Up-regulated").sum() -down_count = (volcano_df["Significance"] == "Down-regulated").sum() +# 6. Keep the existing statistical summary and bottom links +up_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] >= fc_thresh)).sum() +down_count = ((volcano_df["p-adj"] <= p_thresh) & (volcano_df["log2FC"] <= -fc_thresh)).sum() st.markdown(f"**Up-regulated:** {up_count} | **Down-regulated:** {down_count}") st.markdown("---") diff --git a/content/statistical.py b/content/statistical.py new file mode 100644 index 0000000..2e2a46d --- /dev/null +++ b/content/statistical.py @@ -0,0 +1,165 @@ +"""Statistical Inference Page.""" + +from pathlib import Path +import pandas as pd +import polars as pl +import streamlit as st +from src.common.common import page_setup +from src.common.results_helpers import get_abundance_data, get_id_column, get_sample_group_map +# Import statistics engine functions from openms_insight +from openms_insight.analysis.statistics import calculate_statistical_tests, adjust_fdr_lazy + +params = page_setup() +st.title("Statistical Inference") + +st.markdown( + """ +Run differential expression analysis to identify statistically significant proteins across your biological groups. +""" +) + +if "workspace" not in st.session_state: + st.warning("Please initialize your workspace first.") + st.stop() + +# Load primary database assets +result = get_abundance_data(st.session_state["workspace"]) +if result is None: + st.info( + "Abundance data not available. Please run the workflow and configure sample groups first." + ) + st.page_link( + "content/results_abundance.py", label="Go to Abundance", icon="πŸ“‹" + ) + st.stop() + +pivot_df, expr_df, group_map = result +id_col = get_id_column(st.session_state["workspace"], pivot_df) +sample_group_map = get_sample_group_map(st.session_state["workspace"], pivot_df, group_map) + +# --- STEP 1: Upstream Pipeline Tracker (Fallback Architecture) --- +if ( + "normalized_df" in st.session_state + and st.session_state["normalized_df"] is not None +): + base_df = st.session_state["normalized_df"] + st.info( + "πŸ”„ **Upstream Pipeline Detected**: Using data processed from the **Normalization** step." + ) +elif ( + "imputed_df" in st.session_state + and st.session_state["imputed_df"] is not None +): + base_df = st.session_state["imputed_df"] + st.warning( + "⚠️ **Normalization Skipped**: Using data processed from the **Imputation** step." + ) +elif ( + "filtered_df" in st.session_state + and st.session_state["filtered_df"] is not None +): + base_df = st.session_state["filtered_df"] + st.warning( + "⚠️ **Preprocessing Skipped**: Using data processed from the **Filtering** step." + ) +else: + base_df = pivot_df + st.warning( + "⚠️ **Raw Input Active**: No preprocessing history found. Operating on the original table." + ) + +# 2. Extract actual active sample columns and detect unique biological groups +sample_cols = [ + c for c in base_df.columns if c not in [id_col, "PeptideSequence", "log2FC", "p-value", "p-adj"] +] +unique_groups = sorted(list(set([sample_group_map[s] for s in sample_cols if s in sample_group_map]))) +group_count = len(unique_groups) + +# --- SECTION 1: Active Input Table Preview --- +st.subheader("Input Table Overview") +st.markdown( + f"Currently analyzing **{base_df.shape[0]}** rows across **{len(sample_cols)}** samples belonging to **{group_count} groups** ({', '.join(unique_groups)})." +) +st.dataframe(base_df, use_container_width=True) + +st.markdown("---") + +# --- SECTION 2: Dynamic Statistical Parameter Configuration --- +st.subheader("Configure Statistical Engine") + +# Prepare structural Polars metadata DataFrame required by backend functions +metadata_rows = [{"sample_id": s, "group": sample_group_map[s]} for s in sample_cols if s in sample_group_map] +metadata_pl = pl.DataFrame( + metadata_rows, schema={"sample_id": pl.String, "group": pl.String} +) + +col1, col2 = st.columns(2) + +with col1: + st.markdown("### πŸ”¬ 1. Hypothesis Testing Method") + + # Route available method options dynamically based on the group count + if group_count == 2: + method_options = ["limma_like", "welch", "paired"] + help_text = "'limma_like' uses Empirical Bayes variance shrinking. 'welch' is for unequal variances. 'paired' is for dependent samples." + elif group_count >= 3: + method_options = ["limma_like", "anova"] + help_text = "'limma_like' supports multi-group design matrices. 'anova' computes standard row-wise One-way ANOVA." + else: + st.error("❌ Statistical testing requires at least 2 unique sample groups.") + st.stop() + + selected_method = st.selectbox( + "Select Statistical Test", + options=method_options, + index=0, + help=help_text + ) + +with col2: + st.markdown("### πŸ›‘οΈ 2. Multiple Testing Correction (FDR)") + selected_fdr = st.selectbox( + "Select FDR Adjustment Strategy", + options=["BH", "Bonferroni", "None"], + index=0, + help="'BH' (Benjamini-Hochberg) controls False Discovery Rate. 'Bonferroni' is strict Family-Wise Error Rate control." + ) + +# --- SECTION 3: Statistical Query Execution --- +st.markdown("
", unsafe_allow_html=True) +if st.button("Run Statistical Analysis", type="primary"): + + # Convert active pandas dataframe into polars lazyframe graph + stats_lazy = pl.from_pandas(base_df).lazy() + + try: + # Execute Chain 1: Calculate core statistics (Adds log2FC, stat, p-value) + stats_lazy = calculate_statistical_tests( + quantification_data=stats_lazy, + metadata=metadata_pl, + method=selected_method + ) + + # Execute Chain 2: Adjust Multiple Testing (Adds p-adj) + stats_lazy = adjust_fdr_lazy( + quantification_data=stats_lazy, + strategy=selected_fdr + ) + + # Resolve lazy graph optimization tree and bring back to pandas memory + statistics_df = stats_lazy.collect().to_pandas() + + # πŸ’Ύ Save processing checkpoint inside Session State for Downstream (e.g., Volcano plot, Volcano/Heatmap UI) + st.session_state["statistics_df"] = statistics_df + + st.success(f"Successfully calculated **{selected_method}** test with **{selected_fdr}** FDR correction!") + + # Display the finalized statistics table view + st.subheader("Statistical Analysis Results") + st.markdown(f"Generated framework containing columns: `{id_col}`, `log2FC`, `stat`, `p-value`, `p-adj`") + st.dataframe(statistics_df, use_container_width=True) + + except ValueError as val_err: + st.error(f"Engine Validation Fallure: {str(val_err)}") + except Exception as e: + st.error(f"An unexpected pipeline error occurred: {str(e)}") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aac2879..5d20693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -142,6 +142,11 @@ scipy scikit-learn openms-insight>=0.1.13 polars>=1.0.0 +# Forces polars to prefer its most CPU-compatible native runtime. +# Without this, polars can select an AVX-optimized runtime (e.g. runtime-32) +# that access-violation crashes (0xC0000005 in _polars_runtime.pyd) on some +# CPUs (seen on AMD Threadripper PRO 3995WX on Windows) during dataframe ops. +polars-runtime-compat cython easypqp>=0.1.34 pyprophet>=2.2.0 @@ -149,4 +154,5 @@ mygene # Redis Queue dependencies (for online mode) redis>=5.0.0 rq>=1.16.0 -statsmodels \ No newline at end of file +statsmodels +polars \ No newline at end of file diff --git a/src/WorkflowTest.py b/src/WorkflowTest.py index 2af9bda..120efb0 100644 --- a/src/WorkflowTest.py +++ b/src/WorkflowTest.py @@ -1,9 +1,10 @@ import streamlit as st from pathlib import Path +import re import pandas as pd import plotly.express as px -#from streamlit_plotly_events import plotly_events -#from pyopenms import IdXMLFile +from streamlit_plotly_events import plotly_events +from pyopenms import IdXMLFile from scipy.stats import ttest_ind import numpy as np import mygene @@ -14,7 +15,7 @@ from src.common.common import page_setup from src.common.results_helpers import get_abundance_data from src.common.results_helpers import parse_idxml, build_spectra_cache -#from openms_insight import Table, Heatmap, LinePlot, SequenceView +from openms_insight import Table, Heatmap, LinePlot, SequenceView # params = page_setup() class WorkflowTest(WorkflowManager): @@ -47,6 +48,29 @@ def configure(self) -> None: self.ui.select_input_file("mzML-files", multiple=True, reactive=True) self.ui.select_input_file("fasta-file", multiple=False) + self.params = self.parameter_manager.get_parameters_from_json() + saved_mode = self.params.get("analysis-mode", "LFQ") + + self.ui.input_widget( + key="analysis-mode", + default=saved_mode, + name="Analysis Mode", + widget_type="selectbox", + options=["LFQ", "TMT"], + help="Choose between Label-Free Quantification (LFQ) or Tandem Mass Tag (TMT) analysis.", + reactive=True + ) + + self.params = self.parameter_manager.get_parameters_from_json() + current_mode = self.params.get("analysis-mode", "LFQ") + + if current_mode == "LFQ": + self.render_lfq_tabs() + else: + self.render_tmt_tabs() + + def render_lfq_tabs(self): + st.subheader("LFQ Analysis Mode") t = st.tabs(["**Identification**", "**Rescoring**", "**Filtering**", "**Library Generation**", "**Quantification**", "**Group Selection**"]) with t[0]: @@ -70,10 +94,10 @@ def configure(self) -> None: st.info(""" **Decoy Database Settings:** * **method**: How decoy sequences are generated from target protein sequences. - *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly - rearranges the amino acids. Both methods preserve the amino acid composition - of the original protein, ensuring decoys have similar properties to real sequences - for accurate false discovery rate (FDR) estimation. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. """) self.ui.input_TOPP( "DecoyDatabase", @@ -102,7 +126,7 @@ def configure(self) -> None: st.info(comet_info) comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", - "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] if not self.params.get("generate-decoys", True): # Only show decoy_string when not generating decoys comet_include.append("PeptideIndexing:decoy_string") @@ -111,7 +135,7 @@ def configure(self) -> None: "CometAdapter", custom_defaults={ "threads": 8, - "instrument": "high_res", + "instrument": "low_res", "missed_cleavages": 2, "min_peptide_length": 6, "max_peptide_length": 40, @@ -120,17 +144,19 @@ def configure(self) -> None: "isotope_error": "0/1", "precursor_charge": "2:4", "precursor_mass_tolerance": 20.0, - "fragment_mass_tolerance": 0.02, - "fragment_bin_offset": 0.0, + "fragment_mass_tolerance": 0.6, + "fragment_bin_offset": 0.4, "max_variable_mods_in_peptide": 3, "minimum_peaks": 1, "clip_nterm_methionine": "true", - "PeptideIndexing:IL_equivalent": "true", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)", + "PeptideIndexing:IL_equivalent": True, "PeptideIndexing:unmatched_action": "warn", "PeptideIndexing:decoy_string": "rev_", + "mass_recalibration": False, }, - flag_parameters=["PeptideIndexing:IL_equivalent"], include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "mass_recalibration"], exclude_parameters=["second_enzyme"], ) @@ -151,10 +177,10 @@ def configure(self) -> None: "subset_max_train": 300000, "decoy_pattern": "rev_", "score_type": "pep", - "post_processing_tdc": "true", + "post_processing_tdc": True, }, - flag_parameters=["post_processing_tdc"], include_parameters=percolator_include, + flag_parameters=["post_processing_tdc"], exclude_parameters=["out_type"], ) @@ -250,6 +276,10 @@ def configure(self) -> None: "psmFDR": 0.01, "proteinFDR": 0.01, "picked_proteinFDR": "true", + "alignment_order": "star", + "protein_quantification": "unique_peptides", + "quantification_method": "feature_intensity", + "protein_inference": "aggregation", }, include_parameters=["intThreshold", "psmFDR", "proteinFDR"], ) @@ -300,6 +330,294 @@ def configure(self) -> None: if orphaned_keys: self.parameter_manager.save_parameters() + def render_tmt_tabs(self): + st.subheader("TMT Analysis Mode") + # Create tabs for different analysis steps. + t = st.tabs( + ["**IsobaricAnalyzer**", "**CometAdapter**", "**PercolatorAdapter**", "**IDFilter**", "**IDMapper**", "**FileMerger**", + "**ProteinInference**", "**IDFilter**", "**IDConflictResolver**", "**ProteinQuantifier**", "**Group Selection**"] + ) + with t[0]: + # Checkbox for decoy generation + # reactive=True ensures the parent configure() fragment re-runs when checkbox changes, + # so conditional UI (DecoyDatabase settings) updates immediately + self.ui.input_widget( + key="generate-decoys", + default=True, + name="Generate Decoy Database", + widget_type="checkbox", + help="Generate reversed decoy sequences for FDR calculation. Disable if your FASTA already contains decoys.", + reactive=True, + ) + + # Reload params to get current checkbox value after it was saved + self.params = self.parameter_manager.get_parameters_from_json() + + # Show DecoyDatabase settings if generating decoys + if self.params.get("generate-decoys", True): + st.info(""" + **Decoy Database Settings:** + * **method**: How decoy sequences are generated from target protein sequences. + *Reverse* creates decoys by reversing each sequence, while *shuffle* randomly + rearranges the amino acids. Both methods preserve the amino acid composition + of the original protein, ensuring decoys have similar properties to real sequences + for accurate false discovery rate (FDR) estimation. + """) + self.ui.input_TOPP( + "DecoyDatabase", + custom_defaults={ + "decoy_string": "rev_", + "decoy_string_position": "prefix", + "method": "reverse", + }, + include_parameters=["method"], + ) + + comet_info = """ + **Identification (Comet):** + * **enzyme**: The enzyme used for peptide digestion. + * **missed_cleavages**: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + * **fixed_modifications**: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **variable_modifications**: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + * **instrument**: Type of instrument (high_res or low_res). Use 'high_res' for high-resolution MS2 (Orbitrap, TOF), 'low_res' for ion trap. + * **fragment_mass_tolerance**: Fragment mass tolerance for MS2 matching. + * **fragment_bin_offset**: Offset for binning MS2 spectra. Typically 0.0 for high-res, 0.4 for low-res instruments. + """ + if not self.params.get("generate-decoys", True): + comet_info += """* **PeptideIndexing:decoy_string**: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions + in the protein database to indicate decoy proteins. + """ + st.info(comet_info) + + st.write(Path(self.workflow_dir, "results")) + + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset"] + if not self.params.get("generate-decoys", True): + # Only show decoy_string when not generating decoys + comet_include.append("PeptideIndexing:decoy_string") + + self.ui.input_TOPP( + "IsobaricAnalyzer", + custom_defaults={ + "tmt11plex:reference_channel": 126, + "type": "tmt11plex", + "extraction:select_activation": "auto", + "extraction:reporter_mass_shift": 0.002, + "extraction:min_reporter_intensity": 0.0, + "extraction:min_precursor_purity": 0.0, + "extraction:precursor_isotope_deviation": 10.0, + "quantification:isotope_correction": "false", + }, + tool_instance_name="IsobaricAnalyzer-TMT", + reactive=True, + ) + with t[1]: + comet_include = [":enzyme", "missed_cleavages", "fixed_modifications", "variable_modifications", + "instrument", "fragment_mass_tolerance", "fragment_error_units", "fragment_bin_offset", "PeptideIndexing:IL_equivalent"] + self.ui.input_TOPP( + "CometAdapter", + custom_defaults={ + "PeptideIndexing:IL_equivalent": True, + "clip_nterm_methionine": "true", + "instrument": "high_res", + "missed_cleavages": 2, + "min_peptide_length": 6, + "max_peptide_length": 40, + "enzyme": "Trypsin/P", + "PeptideIndexing:unmatched_action": "warn", + "max_variable_mods_in_peptide": 3, + "precursor_mass_tolerance": 4.5, + "isotope_error": "0/1", + "precursor_error_units": "ppm", + "num_hits": 1, + "num_enzyme_termini": "fully", + "fragment_bin_offset": 0.0, + "minimum_peaks": 10, + "precursor_charge": "2:4", + "fragment_mass_tolerance": 0.015, + "PeptideIndexing:unmatched_action": "warn", + "variable_modifications": "Oxidation (M)\nAcetyl (Protein N-term)\nTMT6plex (K)\nTMT6plex (N-term)", + "debug": 0, + "force": True, + }, + include_parameters=comet_include, + flag_parameters=["PeptideIndexing:IL_equivalent", "force"], + exclude_parameters=["second_enzyme"], + tool_instance_name="CometAdapter-TMT", + ) + with t[2]: + st.info(""" + **Filtering (IDFilter):** + * **score:type_peptide**: Score used for filtering. If empty, the main score is used. + * **score:psm**: The score which should be reached by a peptide hit to be kept. (use 'NAN' to disable this filter) + """) + self.ui.input_TOPP( + "PercolatorAdapter", + custom_defaults={ + "subset_max_train": 300000, + "decoy_pattern": "DECOY_", + "score_type": "pep", + "post_processing_tdc": True, + "debug": 0, + }, + flag_parameters=["post_processing_tdc"], + tool_instance_name="PercolatorAdapter-TMT", + ) + + with t[3]: + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_peptide": "q-value", + "score:psm": 0.10, + }, + tool_instance_name="IDFilter-strict", + ) + with t[4]: + st.info(""" + **Quantification (ProteomicsLFQ):** + * **intThreshold**: Peak intensity threshold applied in seed detection. + * **psmFDR**: FDR threshold for sub-protein level (e.g. 0.05=5%). Use -FDR_type to choose the level. Cutoff is applied at the highest level. If Bayesian inference was chosen, it is equivalent with a peptide FDR + * **proteinFDR**: Protein FDR threshold (0.05=5%). + """) + self.ui.input_TOPP( + "IDMapper", + custom_defaults={ + "threads": 8, + "debug": 0, + }, + tool_instance_name="IDMapper-TMT", + ) + with t[5]: + self.ui.input_TOPP( + "FileMerger", + custom_defaults={ + "in_type": "consensusXML", + "append_method": "append_cols", + "annotate_file_origin": True, + "threads": 8, + }, + flag_parameters=["annotate_file_origin"], + tool_instance_name="FileMerger-TMT", + ) + with t[6]: + self.ui.input_TOPP( + "ProteinInference", + custom_defaults={ + "threads": 8, + "picked_decoy_string": "DECOY_", + "picked_fdr": "true", + "protein_fdr": "true", + "Algorithm:use_shared_peptides": "true", + "Algorithm:annotate_indistinguishable_groups": "true", + "Algorithm:score_type": "PEP", + "Algorithm:score_aggregation_method": "best", + "Algorithm:min_peptides_per_protein": 1, + }, + tool_instance_name="ProteinInference-TMT", + ) + with t[7]: + # A single checkbox widget for workflow logic. + # self.ui.input_widget("run-python-script", False, "Run custom Python script") * + # Generate input widgets for a custom Python tool, located at src/python-tools. + # Parameters are specified within the file in the DEFAULTS dictionary. + # self.ui.input_python("example") * + self.ui.input_TOPP( + "IDFilter", + custom_defaults={ + "score:type_protein": "q-value", + "score:proteingroup": 0.01, + "score:psm": 0.01, + "delete_unreferenced_peptide_hits": True, + "remove_decoys": True + }, + flag_parameters=["delete_unreferenced_peptide_hits", "remove_decoys"], + tool_instance_name="IDFilter-lenient", + ) + with t[8]: + self.ui.input_TOPP( + "IDConflictResolver", + custom_defaults={ + "threads": 4, + }, + tool_instance_name="IDConflictResolver-TMT", + ) + + with t[9]: + self.ui.input_TOPP( + "ProteinQuantifier", + custom_defaults={ + "method": "top", + "top:N": 3, + "top:aggregate": "median", + "top:include_all": True, + "ratios": True, + "threads": 8, + "debug": 0, + }, + flag_parameters=["top:include_all", "ratios"], + tool_instance_name="ProteinQuantifier-TMT", + ) + with t[10]: + st.markdown("### πŸ§ͺ TMT Sample Group Assignment") + + latest_params = self.parameter_manager.get_parameters_from_json() + type_key = ( + f"{self.parameter_manager.topp_param_prefix}" + "IsobaricAnalyzer-TMT:1:type" + ) + selected_type = str( + st.session_state.get(type_key) + or latest_params.get("IsobaricAnalyzer-TMT", {}).get("type") + or "tmt11plex" + ).lower() + + m = re.search(r'\d+', selected_type) + is_supported_type = any(label in selected_type for label in ["tmt", "itraq"]) + if not m or not is_supported_type: + st.warning("Please select a supported isobaric type in the IsobaricAnalyzer tab first.") + else: + num_plex = int(m.group()) + channels = [f"sample{i+1}" for i in range(num_plex)] + st.caption(f"Isobaric type: **{selected_type}** - {num_plex} channels") + st.info("Assign a group name to each channel. Use **'skip'** to exclude a channel.") + + for row_start in range(0, num_plex, 2): + c1, c2 = st.columns(2) + + left_idx = row_start + left_channel = channels[left_idx] + with c1: + self.ui.input_widget( + key=f"TMT-group-{left_channel}", + default="", + name=f"Group for channel {left_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + right_idx = row_start + 1 + if right_idx < num_plex: + right_channel = channels[right_idx] + with c2: + self.ui.input_widget( + key=f"TMT-group-{right_channel}", + default="", + name=f"Group for channel {right_idx + 1}", + widget_type="text", + help="e.g. control, case, skip", + ) + + # Remove orphaned params from a previously selected larger plex + self.params = self.parameter_manager.get_parameters_from_json() + valid_keys = {f"TMT-group-{ch}" for ch in channels} + orphaned = [k for k in self.params if k.startswith("TMT-group-") and k not in valid_keys] + if orphaned: + for k in orphaned: + del self.params[k] + self.parameter_manager.save_parameters() + def execution(self) -> bool: """ Refactored TOPP workflow execution: @@ -352,639 +670,944 @@ def execution(self) -> bool: st.info(f"Using original FASTA: {fasta_path.name}") database_fasta = fasta_path - # ================================ - # 1️⃣ Directory setup - # ================================ - results_dir = Path(self.workflow_dir, "results") - comet_dir = results_dir / "comet_results" - perc_dir = results_dir / "percolator_results" - filter_dir = results_dir / "filter_results" - quant_dir = results_dir / "quant_results" - - for d in [comet_dir, perc_dir, filter_dir, quant_dir]: - d.mkdir(parents=True, exist_ok=True) - - self.logger.log("πŸ“ Output directories created") - - # # ================================ - # # 2️⃣ File path definitions (per sample) - # # ================================ - comet_results = [] - percolator_results = [] - filter_results = [] - - for mz in in_mzML: - stem = Path(mz).stem - comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) - percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) - filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + current_mode = self.params.get("analysis-mode", "LFQ") + st.write(f"Current analysis mode: **{current_mode}**") - # ================================ - # 3️⃣ Per-file processing - # ================================ - for i, mz in enumerate(in_mzML): - stem = Path(mz).stem - st.info(f"Processing sample: {stem}") + if current_mode == "LFQ": + self.logger.log("βš™οΈ Running LFQ workflow") - self.logger.log("πŸ”¬ Starting per-sample processing...") + # ================================ + # 1️⃣ Directory setup + # ================================ + results_dir = Path(self.workflow_dir, "results") + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + filter_dir = results_dir / "psm_filter" + quant_dir = results_dir / "quant_results" + + results_dir = Path(self.workflow_dir, "input-files") + + for d in [comet_dir, perc_dir, filter_dir, quant_dir]: + d.mkdir(parents=True, exist_ok=True) + + self.logger.log("πŸ“ Output directories created") + + # ================================ + # 2️⃣ File path definitions (per sample) + # ================================ + comet_results = [] + percolator_results = [] + filter_results = [] + + for mz in in_mzML: + stem = Path(mz).stem + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_per.idXML")) + filter_results.append(str(filter_dir / f"{stem}_filter.idXML")) + + # ================================ + # 3️⃣ Per-file processing + # ================================ + for i, mz in enumerate(in_mzML): + stem = Path(mz).stem + st.info(f"Processing sample: {stem}") + + self.logger.log("πŸ”¬ Starting per-sample processing...") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string - # --- CometAdapter --- - self.logger.log("πŸ”Ž Running peptide search...") - with st.spinner(f"CometAdapter ({stem})"): - comet_extra_params = {"database": str(database_fasta)} - if self.params.get("generate-decoys", True): - # Propagate decoy_string from DecoyDatabase - comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + ): + self.logger.log("Workflow stopped due to error") + return False - if not self.executor.run_topp( - "CometAdapter", - { - "in": in_mzML, - "out": comet_results, - }, - comet_extra_params, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Get fragment tolerance from CometAdapter parameters for visualization - comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") - frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) - frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" - - # Build visualization cache for Comet results - results_dir_path = Path(self.workflow_dir, "results") - cache_dir = results_dir_path / "insight_cache" - cache_dir.mkdir(parents=True, exist_ok=True) - - # Get mzML directory - mzml_dir = Path(in_mzML[0]).parent - - # Build spectra cache (once, shared by all stages) - spectra_df = None - filename_to_index = {} - - for idxml_file in comet_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Build spectra cache (only once) - if spectra_df is None: - filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} - spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - self.logger.log("βœ… Peptide search complete") + self.logger.log("βœ… Peptide search complete") - # --- PercolatorAdapter --- - self.logger.log("πŸ“Š Running rescoring...") - with st.spinner(f"PercolatorAdapter ({stem})"): - if not self.executor.run_topp( - "PercolatorAdapter", - { - "in": comet_results, - "out": percolator_results, - }, - {"decoy_pattern": decoy_string}, # Always propagated from upstream - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Percolator results - for idxml_file in percolator_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + # if not Path(comet_results).exists(): + # st.error(f"CometAdapter failed for {stem}") + # st.stop() - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter ({stem})"): + if not self.executor.run_topp( + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + {"decoy_pattern": decoy_string}, # Always propagated from upstream + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - self.logger.log("βœ… Rescoring complete") + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # if not Path(percolator_results[i]).exists(): - # st.error(f"PercolatorAdapter failed for {stem}") - # st.stop() + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - # --- IDFilter --- - self.logger.log("πŸ”§ Filtering identifications...") - with st.spinner(f"IDFilter ({stem})"): - if not self.executor.run_topp( - "IDFilter", - { - "in": percolator_results, - "out": filter_results, - }, - ): - self.logger.log("Workflow stopped due to error") - return False - - # Build visualization cache for Filter results - for idxml_file in filter_results: - idxml_path = Path(idxml_file) - cache_id_prefix = idxml_path.stem - - # Parse idXML to DataFrame - id_df, spectra_data = parse_idxml(idxml_path) - - # Initialize Table component (caches itself) - Table( - cache_id=f"table_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, - column_definitions=[ - {"field": "sequence", "title": "Sequence"}, - {"field": "charge", "title": "Z", "sorter": "number"}, - {"field": "mz", "title": "m/z", "sorter": "number"}, - {"field": "rt", "title": "RT", "sorter": "number"}, - {"field": "score", "title": "Score", "sorter": "number"}, - {"field": "protein_accession", "title": "Proteins"}, - ], - initial_sort=[{"column": "score", "dir": "asc"}], - index_field="id_idx", - ) + self.logger.log("βœ… Rescoring complete") - # Initialize Heatmap component - Heatmap( - cache_id=f"heatmap_{cache_id_prefix}", - data=id_df.lazy(), - cache_path=str(cache_dir), - x_column="rt", - y_column="mz", - intensity_column="score", - interactivity={"identification": "id_idx"}, - ) + # if not Path(percolator_results[i]).exists(): + # st.error(f"PercolatorAdapter failed for {stem}") + # st.stop() - # Initialize SequenceView component - seq_view = SequenceView( - cache_id=f"seqview_{cache_id_prefix}", - sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ - "id_idx": "sequence_id", - "charge": "precursor_charge", - }), - peaks_data=spectra_df.lazy(), - filters={ - "identification": "sequence_id", - "file": "file_index", - "spectrum": "scan_id", - }, - interactivity={"peak": "peak_id"}, - cache_path=str(cache_dir), - deconvolved=False, - annotation_config={ - "ion_types": ["b", "y"], - "neutral_losses": True, - "tolerance": frag_tol, - "tolerance_ppm": frag_tol_is_ppm, - }, - ) + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter ({stem})"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": filter_results, + }, + ): + self.logger.log("Workflow stopped due to error") + return False - # Initialize LinePlot from SequenceView - LinePlot.from_sequence_view( - seq_view, - cache_id=f"lineplot_{cache_id_prefix}", - cache_path=str(cache_dir), - title="Annotated Spectrum", - styling={ - "unhighlightedColor": "#CCCCCC", - "highlightColor": "#E74C3C", - "selectedColor": "#F3A712", - }, - ) + # Build visualization cache for Filter results + for idxml_file in filter_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - self.logger.log("βœ… Filtering complete") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - # if not Path(filter_results[i]).exists(): - # st.error(f"IDFilter failed for {stem}") - # st.stop() + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # ================================ - # EasyPQP Spectral Library Generation (optional) - # ================================ - if self.params.get("generate-library", False): - self.logger.log("πŸ“š Building spectral library with EasyPQP...") - st.info("Building spectral library with EasyPQP...") - library_dir = Path(self.workflow_dir, "results", "library") - library_dir.mkdir(parents=True, exist_ok=True) - - psms_files, peaks_files = [], [] - - for filter_idxml in filter_results: - original_stem = Path(filter_idxml).stem.replace("_filter", "") - matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) - if not matching_mzml: - self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") - continue - - # easypqp library requires specific extensions for file recognition: - # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension - # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension - # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name - psms_out = str(library_dir / f"{original_stem}.psmpkl") - peaks_out = str(library_dir / f"{original_stem}.peakpkl") - - convert_cmd = [ - "easypqp", "convert", - "--pepxml", filter_idxml, - "--spectra", matching_mzml, - "--psms", psms_out, - "--peaks", peaks_out - ] - if self.executor.run_command(convert_cmd): - psms_files.append(psms_out) - peaks_files.append(peaks_out) - - if psms_files: - # easypqp library outputs TSV format (despite common .pqp extension) - library_tsv = str(library_dir / "spectral_library.tsv") - library_cmd = ["easypqp", "library", "--out", library_tsv] - - if not self.params.get("library-use-fdr", False): - # --nofdr only skips FDR recalculation, NOT threshold filtering - # Set all thresholds to 1.0 to bypass filtering for pre-filtered input - library_cmd.extend([ - "--nofdr", - "--psm_fdr_threshold", "1.0", - "--peptide_fdr_threshold", "1.0", - "--protein_fdr_threshold", "1.0" - ]) - else: - # Apply user-specified FDR filtering - library_cmd.extend([ - "--psm_fdr_threshold", - str(self.params.get("library-psm-fdr", 0.01)), - "--peptide_fdr_threshold", - str(self.params.get("library-peptide-fdr", 0.01)), - "--protein_fdr_threshold", - str(self.params.get("library-protein-fdr", 0.01)) - ]) - - for psms, peaks in zip(psms_files, peaks_files): - library_cmd.extend([psms, peaks]) - - if self.executor.run_command(library_cmd): - self.logger.log("βœ… Spectral library created") - st.success("Spectral library created") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + self.logger.log("βœ… Filtering complete") + + # if not Path(filter_results[i]).exists(): + # st.error(f"IDFilter failed for {stem}") + # st.stop() + + # ================================ + # EasyPQP Spectral Library Generation (optional) + # ================================ + if self.params.get("generate-library", False): + self.logger.log("πŸ“š Building spectral library with EasyPQP...") + st.info("Building spectral library with EasyPQP...") + library_dir = Path(self.workflow_dir, "results", "library") + library_dir.mkdir(parents=True, exist_ok=True) + + psms_files, peaks_files = [], [] + + for filter_idxml in filter_results: + original_stem = Path(filter_idxml).stem.replace("_filter", "") + matching_mzml = next((m for m in in_mzML if Path(m).stem == original_stem), None) + if not matching_mzml: + self.logger.log(f"Warning: No matching mzML found for {filter_idxml}") + continue + + # easypqp library requires specific extensions for file recognition: + # - PSM files must contain 'psmpkl' β†’ use .psmpkl extension + # - Peak files must contain 'peakpkl' β†’ use .peakpkl extension + # After splitext(), stem will be just "{mzML_stem}" matching PSM base_name + psms_out = str(library_dir / f"{original_stem}.psmpkl") + peaks_out = str(library_dir / f"{original_stem}.peakpkl") + + convert_cmd = [ + "easypqp", "convert", + "--pepxml", filter_idxml, + "--spectra", matching_mzml, + "--psms", psms_out, + "--peaks", peaks_out + ] + if self.executor.run_command(convert_cmd): + psms_files.append(psms_out) + peaks_files.append(peaks_out) + + if psms_files: + # easypqp library outputs TSV format (despite common .pqp extension) + library_tsv = str(library_dir / "spectral_library.tsv") + library_cmd = ["easypqp", "library", "--out", library_tsv] + + if not self.params.get("library-use-fdr", False): + # --nofdr only skips FDR recalculation, NOT threshold filtering + # Set all thresholds to 1.0 to bypass filtering for pre-filtered input + library_cmd.extend([ + "--nofdr", + "--psm_fdr_threshold", "1.0", + "--peptide_fdr_threshold", "1.0", + "--protein_fdr_threshold", "1.0" + ]) + else: + # Apply user-specified FDR filtering + library_cmd.extend([ + "--psm_fdr_threshold", + str(self.params.get("library-psm-fdr", 0.01)), + "--peptide_fdr_threshold", + str(self.params.get("library-peptide-fdr", 0.01)), + "--protein_fdr_threshold", + str(self.params.get("library-protein-fdr", 0.01)) + ]) + + for psms, peaks in zip(psms_files, peaks_files): + library_cmd.extend([psms, peaks]) + + if self.executor.run_command(library_cmd): + self.logger.log("βœ… Spectral library created") + st.success("Spectral library created") + else: + self.logger.log("Warning: Failed to build spectral library") else: - self.logger.log("Warning: Failed to build spectral library") - else: - self.logger.log("Warning: No PSMs converted for library generation") + self.logger.log("Warning: No PSMs converted for library generation") + + st.success(f"βœ“ {stem} identification completed") + + # # ================================ + # # 4️⃣ ProteomicsLFQ (cross-sample) + # # ================================ + self.logger.log("πŸ“ˆ Running cross-sample quantification...") + st.info("Running ProteomicsLFQ (cross-sample quantification)") + + quant_mztab = str(quant_dir / "openms_quant.mzTab") + quant_cxml = str(quant_dir / "openms.consensusXML") + quant_msstats = str(quant_dir / "openms_msstats.csv") + + with st.spinner("ProteomicsLFQ"): + combined_in = " ".join(in_mzML) + combined_ids = " ".join(filter_results) + self.logger.log(f"COMBINED_IN {combined_in}", 1) + self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) + self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) + self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + + # βœ… Streamlit output (debug view) + st.markdown("### πŸ” ProteomicsLFQ Input Debug") + st.write("**combined_in:**", combined_in) + st.write("**combined_in type:**", type(combined_in).__name__) + + st.write("**combined_ids:**", combined_ids) + st.write("**combined_ids type:**", type(combined_ids).__name__) + + if not self.executor.run_topp( + "ProteomicsLFQ", + { + "in": [in_mzML], + "ids": [filter_results], + "out": [quant_mztab], + "out_cxml": [quant_cxml], + "out_msstats": [quant_msstats], + }, + { + "fasta": str(database_fasta), + "threads": 12, + # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 + "PeptideQuantification:extract:IM_window": "0.0", + "PeptideQuantification:faims:merge_features": "false", + }, + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… Quantification complete") + + # if not Path(quant_mztab).exists(): + # st.error("ProteomicsLFQ failed: mzTab not created") + # st.stop() + + # ================================ + # 5️⃣ Final report + # # ================================ + st.success("πŸŽ‰ TOPP workflow completed successfully") + st.write("πŸ“ Results directory:") + st.code(str(results_dir)) + + st.write("πŸ“„ Generated files:") + st.write(f"- mzTab: {quant_mztab}") + st.write(f"- consensusXML: {quant_cxml}") + st.write(f"- MSstats CSV: {quant_msstats}") - st.success(f"βœ“ {stem} identification completed") + return True + else: + self.logger.log("βš™οΈ Running TMT workflow") - # ================================ - # 4️⃣ ProteomicsLFQ (cross-sample) - # ================================ - self.logger.log("πŸ“ˆ Running cross-sample quantification...") - st.info("Running ProteomicsLFQ (cross-sample quantification)") + results_dir = Path(self.workflow_dir, "results") + iso_dir = results_dir / "isobaric_consensusXML" + comet_dir = results_dir / "comet_results" + perc_dir = results_dir / "percolator_results" + psm_filter_dir = results_dir / "psm_filter" + map_dir = results_dir / "idmapper" + merge_dir = results_dir / "merged" + protein_dir = results_dir / "protein" + msstats_dir = results_dir / "msstats" + quant_dir = results_dir / "quant_results" + + iso_consensus = [] + comet_results = [] + percolator_results = [] + psm_filtered = [] + mapped_ids = [] + + for d in [ + iso_dir, comet_dir, perc_dir, psm_filter_dir, + map_dir, merge_dir, protein_dir, msstats_dir, quant_dir + ]: + d.mkdir(parents=True, exist_ok=True) + + for mz in in_mzML: + stem = Path(mz).stem + iso_consensus.append(str(iso_dir / f"{stem}_iso.consensusXML")) + comet_results.append(str(comet_dir / f"{stem}_comet.idXML")) + percolator_results.append(str(perc_dir / f"{stem}_comet_perc.idXML")) + psm_filtered.append(str(psm_filter_dir / f"{stem}_comet_perc_filter.idXML")) + mapped_ids.append(str(map_dir / f"{stem}_comet_perc_filter_map.consensusXML")) + + merged_id = str(merge_dir / "ID_mapper_merge.consensusXML") + protein_id = str(protein_dir / "ID_mapper_merge_epi.consensusXML") + protein_filter = str(protein_dir / "ID_mapper_merge_epi_filter.consensusXML") + protein_resolved = str(protein_dir / "ID_mapper_merge_epi_filter_resconf.consensusXML") + consensus_out = str(quant_dir / "openms_design_protein_openms.csv") + + # --- IsobaricAnalyzer --- + self.logger.log("🏷️ Running isobaric labeling analysis...") + with st.spinner("IsobaricAnalyzer"): + if not self.executor.run_topp( + "IsobaricAnalyzer", + { + "in": in_mzML, + "out": iso_consensus, + }, + tool_instance_name="IsobaricAnalyzer-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IsobaricAnalyzer complete") + + # --- CometAdapter --- + self.logger.log("πŸ”Ž Running peptide search...") + with st.spinner(f"CometAdapter ({stem})"): + comet_extra_params = {"database": str(database_fasta)} + if self.params.get("generate-decoys", True): + # Propagate decoy_string from DecoyDatabase + comet_extra_params["PeptideIndexing:decoy_string"] = decoy_string + if not self.executor.run_topp( + "CometAdapter", + { + "in": in_mzML, + "out": comet_results, + }, + comet_extra_params, + tool_instance_name="CometAdapter-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… CometAdapter complete") + + # Get fragment tolerance from CometAdapter parameters for visualization + comet_params = self.parameter_manager.get_topp_parameters("CometAdapter") + frag_tol = comet_params.get("fragment_mass_tolerance", 0.02) + frag_tol_is_ppm = comet_params.get("fragment_error_units", "Da") != "Da" + + # Build visualization cache for Comet results + results_dir_path = Path(self.workflow_dir, "results") + cache_dir = results_dir_path / "insight_cache" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Get mzML directory + mzml_dir = Path(in_mzML[0]).parent + + # Build spectra cache (once, shared by all stages) + spectra_df = None + filename_to_index = {} + + for idxml_file in comet_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Build spectra cache (only once) + if spectra_df is None: + filename_to_index = {Path(f).name: i for i, f in enumerate(spectra_data)} + spectra_df, filename_to_index = build_spectra_cache(mzml_dir, filename_to_index) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - quant_mztab = str(quant_dir / "openms_quant.mzTab") - quant_cxml = str(quant_dir / "openms.consensusXML") - quant_msstats = str(quant_dir / "openms_msstats.csv") + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - with st.spinner("ProteomicsLFQ"): - combined_in = " ".join(in_mzML) - combined_ids = " ".join(filter_results) - self.logger.log(f"COMBINED_IN {combined_in}", 1) - self.logger.log(f"COMBINED_IN_TYPE {type(combined_in).__name__}", 1) - self.logger.log(f"FILTER_RESULTS = {filter_results}", 1) - self.logger.log(f"FILTER_RESULTS_LEN = {len(filter_results)}", 1) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - # βœ… Streamlit output (debug view) - st.markdown("### πŸ” ProteomicsLFQ Input Debug") - st.write("**combined_in:**", combined_in) - st.write("**combined_in type:**", type(combined_in).__name__) + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - st.write("**combined_ids:**", combined_ids) - st.write("**combined_ids type:**", type(combined_ids).__name__) + self.logger.log("βœ… Peptide search complete") + # --- PercolatorAdapter --- + self.logger.log("πŸ“Š Running rescoring...") + with st.spinner(f"PercolatorAdapter"): if not self.executor.run_topp( - "ProteomicsLFQ", - { - "in": [in_mzML], - "ids": [filter_results], - "out": [quant_mztab], - "out_cxml": [quant_cxml], - "out_msstats": [quant_msstats], - }, - { - "fasta": str(database_fasta), - "psmFDR": 0.5, - "proteinFDR": 0.5, - "threads": 12, - # Disable FAIMS/IM handling to avoid segfault in OpenMS 3.5.0 - "PeptideQuantification:extract:IM_window": "0.0", - "PeptideQuantification:faims:merge_features": "false", - } - ): + "PercolatorAdapter", + { + "in": comet_results, + "out": percolator_results, + }, + tool_instance_name="PercolatorAdapter-TMT", + ): self.logger.log("Workflow stopped due to error") return False - self.logger.log("βœ… Quantification complete") - - # ====================================================== - # ⚠️ 5️⃣ GO Enrichment Analysis (INLINE IN EXECUTION) - # ====================================================== - workspace_path = Path(self.workflow_dir).parent - res = get_abundance_data(workspace_path) - if res is not None: - pivot_df, _, _ = res - self.logger.log("βœ… pivot_df loaded, starting GO enrichment...") - self._run_go_enrichment(pivot_df, results_dir) - else: - st.warning("GO enrichment skipped: abundance data not available.") + # Build visualization cache for Percolator results + for idxml_file in percolator_results: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - # ================================ - # 5️⃣ Final report - # # ================================ - st.success("πŸŽ‰ TOPP workflow completed successfully") - st.write("πŸ“ Results directory:") - st.code(str(results_dir)) - - return True - - def _run_go_enrichment(self, pivot_df: pd.DataFrame, results_dir: Path): - p_cutoff = 0.05 - fc_cutoff = 1.0 - - analysis_df = pivot_df.dropna(subset=["p-value", "log2FC"]).copy() - - if analysis_df.empty: - st.error("No valid statistical data found for GO enrichment.") - self.logger.log("❗ analysis_df is empty") - else: - with st.spinner("Fetching GO terms from MyGene.info API..."): - mg = mygene.MyGeneInfo() - - def get_clean_uniprot(name): - parts = str(name).split("|") - return parts[1] if len(parts) >= 2 else parts[0] - - analysis_df["UniProt"] = analysis_df["ProteinName"].apply(get_clean_uniprot) - - bg_ids = analysis_df["UniProt"].dropna().astype(str).unique().tolist() - fg_ids = analysis_df[ - (analysis_df["p-value"] < p_cutoff) & - (analysis_df["log2FC"].abs() >= fc_cutoff) - ]["UniProt"].dropna().astype(str).unique().tolist() - self.logger.log("βœ… get_clean_uniprot applied") - - if len(fg_ids) < 3: - st.warning( - f"Not enough significant proteins " - f"(p < {p_cutoff}, |log2FC| β‰₯ {fc_cutoff}). " - f"Found: {len(fg_ids)}" - ) - self.logger.log("❗ Not enough significant proteins") - else: - res_list = mg.querymany( - bg_ids, scopes="uniprot", fields="go", as_dataframe=False - ) - res_go = pd.DataFrame(res_list) - if "notfound" in res_go.columns: - res_go = res_go[res_go["notfound"] != True] - - def extract_go_terms(go_data, go_type): - if not isinstance(go_data, dict) or go_type not in go_data: - return [] - terms = go_data[go_type] - if isinstance(terms, dict): - terms = [terms] - return list({t.get("term") for t in terms if "term" in t}) - - for go_type in ["BP", "CC", "MF"]: - res_go[f"{go_type}_terms"] = res_go["go"].apply( - lambda x: extract_go_terms(x, go_type) - ) + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - annotated_ids = set(res_go["query"].astype(str)) - fg_set = annotated_ids.intersection(fg_ids) - bg_set = annotated_ids - self.logger.log(f"βœ… fg_set bg_set are set") - - def run_go(go_type): - go2fg = defaultdict(set) - go2bg = defaultdict(set) - - for _, row in res_go.iterrows(): - uid = str(row["query"]) - for term in row[f"{go_type}_terms"]: - go2bg[term].add(uid) - if uid in fg_set: - go2fg[term].add(uid) - - records = [] - N_fg = len(fg_set) - N_bg = len(bg_set) - - for term, fg_genes in go2fg.items(): - a = len(fg_genes) - if a == 0: - continue - b = N_fg - a - c = len(go2bg[term]) - a - d = N_bg - (a + b + c) - - _, p = fisher_exact([[a, b], [c, d]], alternative="greater") - records.append({ - "GO_Term": term, - "Count": a, - "GeneRatio": f"{a}/{N_fg}", - "p_value": p, - }) - - df = pd.DataFrame(records) - if df.empty: - return None, None - - df["-log10(p)"] = -np.log10(df["p_value"].replace(0, 1e-10)) - df = df.sort_values("p_value").head(20) - - # βœ… Plotly Figure - fig = px.bar( - df, - x="-log10(p)", - y="GO_Term", - orientation="h", - title=f"GO Enrichment ({go_type})", - ) + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - self.logger.log(f"βœ… Plotly Figure generated") + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) - fig.update_layout( - yaxis=dict(autorange="reversed"), - height=500, - margin=dict(l=10, r=10, t=40, b=10), - ) + self.logger.log("βœ… PercolatorAdapter complete") + + # --- IDFilter --- + self.logger.log("πŸ”§ Filtering identifications...") + with st.spinner(f"IDFilter"): + if not self.executor.run_topp( + "IDFilter", + { + "in": percolator_results, + "out": psm_filtered, + }, + tool_instance_name="IDFilter-strict" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-strict complete") + + # Build visualization cache for Filter results + for idxml_file in psm_filtered: + idxml_path = Path(idxml_file) + cache_id_prefix = idxml_path.stem + + # Parse idXML to DataFrame + id_df, spectra_data = parse_idxml(idxml_path) + + # Initialize Table component (caches itself) + Table( + cache_id=f"table_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + interactivity={"file": "file_index", "spectrum": "scan_id", "identification": "id_idx"}, + column_definitions=[ + {"field": "sequence", "title": "Sequence"}, + {"field": "charge", "title": "Z", "sorter": "number"}, + {"field": "mz", "title": "m/z", "sorter": "number"}, + {"field": "rt", "title": "RT", "sorter": "number"}, + {"field": "score", "title": "Score", "sorter": "number"}, + {"field": "protein_accession", "title": "Proteins"}, + ], + initial_sort=[{"column": "score", "dir": "asc"}], + index_field="id_idx", + ) - return fig, df + # Initialize Heatmap component + Heatmap( + cache_id=f"heatmap_{cache_id_prefix}", + data=id_df.lazy(), + cache_path=str(cache_dir), + x_column="rt", + y_column="mz", + intensity_column="score", + interactivity={"identification": "id_idx"}, + ) - go_results = {} + # Initialize SequenceView component + seq_view = SequenceView( + cache_id=f"seqview_{cache_id_prefix}", + sequence_data=id_df.lazy().select(["id_idx", "sequence", "charge", "file_index", "scan_id"]).rename({ + "id_idx": "sequence_id", + "charge": "precursor_charge", + }), + peaks_data=spectra_df.lazy(), + filters={ + "identification": "sequence_id", + "file": "file_index", + "spectrum": "scan_id", + }, + interactivity={"peak": "peak_id"}, + cache_path=str(cache_dir), + deconvolved=False, + annotation_config={ + "ion_types": ["b", "y"], + "neutral_losses": True, + "tolerance": frag_tol, + "tolerance_ppm": frag_tol_is_ppm, + }, + ) - for go_type in ["BP", "CC", "MF"]: - fig, df_go = run_go(go_type) - if fig is not None: - go_results[go_type] = { - "fig": fig, - "df": df_go - } - self.logger.log(f"βœ… go_type generated") - - go_dir = results_dir / "go-terms" - go_dir.mkdir(parents=True, exist_ok=True) - - import json - go_data = {} - - for go_type in ["BP", "CC", "MF"]: - if go_type in go_results: - fig = go_results[go_type]["fig"] - df = go_results[go_type]["df"] - - go_data[go_type] = { - "fig_json": fig.to_json(), # Figure β†’ JSON string - "df_dict": df.to_dict(orient="records") # DataFrame β†’ list of dicts - } - - go_json_file = go_dir / "go_results.json" - with open(go_json_file, "w") as f: - json.dump(go_data, f) - st.session_state["go_results"] = go_results - st.session_state["go_ready"] = True if go_data else False - self.logger.log("βœ… GO enrichment analysis complete") - + # Initialize LinePlot from SequenceView + LinePlot.from_sequence_view( + seq_view, + cache_id=f"lineplot_{cache_id_prefix}", + cache_path=str(cache_dir), + title="Annotated Spectrum", + styling={ + "unhighlightedColor": "#CCCCCC", + "highlightColor": "#E74C3C", + "selectedColor": "#F3A712", + }, + ) + + # --- IDMapper --- + self.logger.log("πŸ—ΊοΈ Mapping IDs to isobaric consensus features...") + for iso, psm, mapped in zip(iso_consensus, psm_filtered, mapped_ids): + iso_stem = Path(iso).stem + with st.spinner(f"IDMapper ({iso_stem})"): + if not self.executor.run_topp( + "IDMapper", + { + "in": [iso], + "id": [psm], + "out": [mapped], + }, + tool_instance_name="IDMapper-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDMapper complete") + + # --- FileMerger --- + self.logger.log("πŸ”— Merging mapped consensus files...") + with st.spinner("FileMerger"): + if not self.executor.run_topp( + "FileMerger", + { + "in": mapped_ids, + "out": [merged_id], + }, + tool_instance_name="FileMerger-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… FileMerger complete") + + # --- ProteinInference --- + self.logger.log("🧩 Running protein inference...") + with st.spinner("ProteinInference"): + if not self.executor.run_topp( + "ProteinInference", + { + "in": [merged_id], + "out": [protein_id], + }, + tool_instance_name="ProteinInference-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinInference complete") + + # --- IDFilter-lenient (Protein) --- + self.logger.log("πŸ”¬ Filtering proteins...") + with st.spinner("IDFilter (Protein)"): + if not self.executor.run_topp( + "IDFilter", + { + "in": [protein_id], + "out": [protein_filter], + }, + tool_instance_name="IDFilter-lenient" + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDFilter-lenient (Protein) complete") + + # ================================ + # ✨ NEW: 8️⃣ IDConflictResolver (protein_filter β†’ protein_resolved) + # ================================ + self.logger.log("βš–οΈ Resolving ID conflicts...") + with st.spinner("IDConflictResolver"): + if not self.executor.run_topp( + "IDConflictResolver", + { + "in": [protein_filter], + "out": [protein_resolved], + }, + tool_instance_name="IDConflictResolver-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… IDConflictResolver complete") + + # ================================ + # ✨ NEW: πŸ”Ÿ ProteinQuantifier (protein_resolved β†’ consensus_out) + # ================================ + self.logger.log("πŸ“ Running protein quantification...") + with st.spinner("ProteinQuantifier"): + if not self.executor.run_topp( + "ProteinQuantifier", + { + "in": [protein_resolved], + "out": [consensus_out], + }, + tool_instance_name="ProteinQuantifier-TMT", + ): + self.logger.log("Workflow stopped due to error") + return False + self.logger.log("βœ… ProteinQuantifier complete") + self.logger.log("πŸ“„ Generating protein table...") + + self.logger.log("πŸŽ‰ WORKFLOW FINISHED") @st.fragment def results(self) -> None: diff --git a/src/common/results_helpers.py b/src/common/results_helpers.py index db3e103..2d38ad9 100644 --- a/src/common/results_helpers.py +++ b/src/common/results_helpers.py @@ -5,10 +5,8 @@ import numpy as np import streamlit as st from pathlib import Path -from scipy.stats import ttest_ind from pyopenms import IdXMLFile, MSExperiment, MzMLFile from src.workflow.ParameterManager import ParameterManager -from statsmodels.stats.multitest import multipletests def get_workflow_dir(workspace): """Get the workflow directory path.""" @@ -184,12 +182,15 @@ def build_spectra_cache(mzml_dir: Path, filename_to_index: dict) -> tuple[pl.Dat @st.cache_data -def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: - """Load CSV, compute stats (log2FC, p-value), build pivot_df and expr_df. +def load_abundance_data(workspace_path: str, csv_mtime: float, params_mtime: float = 0.0) -> tuple | None: + """Load CSV and build abundance matrices for downstream preprocessing. Args: workspace_path: Path to the workspace directory csv_mtime: Modification time of CSV file (used as cache key) + params_mtime: Modification time of params.json (used as cache key so + changing group assignments in Configure invalidates the cache + even when the CSV itself hasn't changed) Returns: Tuple of (pivot_df, expr_df, group_map) or None if data unavailable @@ -197,115 +198,166 @@ def load_abundance_data(workspace_path: str, csv_mtime: float) -> tuple | None: workflow_dir = get_workflow_dir(Path(workspace_path)) quant_dir = workflow_dir / "results" / "quant_results" - if not quant_dir.exists(): - return None - - csv_files = sorted(quant_dir.glob("*.csv")) - if not csv_files: - return None - - csv_file = csv_files[0] - - try: - df = pd.read_csv(csv_file) - except Exception: - return None + parameter_manager = ParameterManager(workflow_dir, "TOPP Workflow") - if df.empty: - return None + workflow_params = parameter_manager.get_parameters_from_json() + analysis_mode = workflow_params.get("analysis-mode", "LFQ") - # Get group mapping from parameters - param_manager = ParameterManager(workflow_dir) - params = param_manager.get_parameters_from_json() - group_map = { - key[11:]: value # Remove "mzML-group-" prefix - for key, value in params.items() - if key.startswith("mzML-group-") and value - } + if analysis_mode == "LFQ": + if not quant_dir.exists(): + return None - if not group_map: - return None + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None - df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) - df["Group"] = df["Reference"].map(group_map) - df = df.dropna(subset=["Group"]) + csv_file = csv_files[0] - groups = sorted(df["Group"].unique()) + try: + df = pd.read_csv(csv_file) + except Exception: + return None - if len(groups) < 2: - return None + if df.empty: + return None - group1, group2 = groups[:2] - - # Compute statistics per protein - stats_rows = [] - for protein, protein_df in df.groupby("ProteinName"): - g1_vals = protein_df[protein_df["Group"] == group1]["Intensity"].values - g2_vals = protein_df[protein_df["Group"] == group2]["Intensity"].values + # Get optional group mapping from parameters. + # Group information is not required at this stage; statistical testing + # happens in the Statistical page. + param_manager = ParameterManager(workflow_dir) + params = param_manager.get_parameters_from_json() + group_map = { + key[11:]: value # Remove "mzML-group-" prefix + for key, value in params.items() + if key.startswith("mzML-group-") and value + } - if len(g1_vals) < 2 or len(g2_vals) < 2: - pval = np.nan + df["Sample"] = df["Reference"].str.replace(".mzML", "", regex=False) + + # Build sample display order. + if group_map: + sample_group_df = df[["Sample", "Reference"]].drop_duplicates() + sample_group_df["Group"] = sample_group_df["Reference"].map(group_map) + grouped_samples = [] + for grp in sorted(sample_group_df["Group"].dropna().unique()): + grouped_samples.extend( + sample_group_df[sample_group_df["Group"] == grp]["Sample"].tolist() + ) + remaining_samples = [ + s for s in sorted(df["Sample"].unique()) if s not in grouped_samples + ] + all_samples = grouped_samples + remaining_samples else: - _, pval = ttest_ind(g1_vals, g2_vals, equal_var=False) - - mean_g1 = np.mean(g1_vals) if len(g1_vals) > 0 else np.nan - mean_g2 = np.mean(g2_vals) if len(g2_vals) > 0 else np.nan - - log2fc = np.log2(mean_g2 / mean_g1) if mean_g1 > 0 else np.nan + all_samples = sorted(df["Sample"].unique()) + + # Build pivot table + pivot_list = [] + for protein, group_df in df.groupby("ProteinName"): + peptides = ";".join(group_df["PeptideSequence"].unique()) + intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() + intensity_dict_complete = { + sample: intensity_dict.get(sample, 0) + for sample in all_samples + } + row = { + "ProteinName": protein, + **intensity_dict_complete, + "PeptideSequence": peptides, + } + pivot_list.append(row) + + pivot_df = pd.DataFrame(pivot_list) + pivot_df = pivot_df[["ProteinName"] + all_samples + ["PeptideSequence"]] + + # Build expression matrix (log2-transformed) + expr_df = pivot_df.set_index("ProteinName")[all_samples] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() + + return pivot_df, expr_df, group_map + + else: + if not quant_dir.exists(): + return None + + csv_files = sorted(quant_dir.glob("*.csv")) + if not csv_files: + return None + + csv_file = csv_files[0] + + try: + df = pd.read_csv(csv_file, sep="\t", comment="#", engine="python") + except Exception: + return None + + if df.empty: + return None + + # ratio column removal + df = df.loc[:, ~df.columns.str.contains('ratio', case=False)] + + # exclude_indices = st.session_state.get("tmt_exclude_indices", []) + # group_map = st.session_state.get("tmt_group_map", {}) + # Get group mapping from parameters + parameter_manager = ParameterManager(Path(workflow_dir), "TOPP Workflow") + params = parameter_manager.get_parameters_from_json() + group_map = {} + for key, value in params.items(): + if key.startswith("TMT-group-") and value: + # Extract the numeric part from keys like "TMT-group-sample1" + match = re.search(r'sample(\d+)', key) + if match: + # Subtract 1 to convert to a 0-based index (0, 1, 2...). + # If your samples are already 0-based, remove the -1 adjustment. + index = str(int(match.group(1)) - 1) + group_map[index] = value + + # 1. Extract keys labeled as "skip" from group_map as integer list + exclude_indices = [ + int(k) for k, v in group_map.items() if v.lower() == "skip" + ] + + # 2. Remove "skip" entries from group_map (keep only actual group info) + group_map = { + int(k): v for k, v in group_map.items() if v.lower() != "skip" + } - stats_rows.append({ - "ProteinName": protein, - "log2FC": log2fc, - "p-value": pval, - }) + start_column_offset = 4 - stats_df = pd.DataFrame(stats_rows) + # st.write("exclude_indices:", exclude_indices) + # st.write("group_map:", group_map) - if not stats_df.empty: - mask = stats_df["p-value"].notna() - if mask.any(): - _, p_adj, _, _ = multipletests(stats_df.loc[mask, "p-value"], method="fdr_bh") - stats_df.loc[mask, "p-adj"] = p_adj + if exclude_indices: + # st.write("Current columns:", df.columns.tolist()) + # st.write("Number of columns:", len(df.columns)) + # st.write("Exclude indices:", exclude_indices) + # st.write("Offset:", start_column_offset) + cols_to_drop = [df.columns[i + start_column_offset] for i in exclude_indices] + df_cleaned = df.drop(columns=cols_to_drop) else: - stats_df["p-adj"] = np.nan - - # Order samples by group (group2 first, then group1) - sample_group_df = df[["Sample", "Group"]].drop_duplicates() - group2_samples = sample_group_df[sample_group_df["Group"] == group2]["Sample"].tolist() - group1_samples = sample_group_df[sample_group_df["Group"] == group1]["Sample"].tolist() - all_samples = group2_samples + group1_samples - - # Build pivot table - pivot_list = [] - for protein, group_df in df.groupby("ProteinName"): - peptides = ";".join(group_df["PeptideSequence"].unique()) - intensity_dict = group_df.groupby("Sample")["Intensity"].sum().to_dict() - intensity_dict_complete = { - sample: intensity_dict.get(sample, 0) - for sample in all_samples - } - row = { - "ProteinName": protein, - **intensity_dict_complete, - "PeptideSequence": peptides, - } - pivot_list.append(row) + df_cleaned = df.copy() + + current_cols = df_cleaned.columns.tolist() + sample_cols = current_cols[start_column_offset:] - pivot_df = pd.DataFrame(pivot_list) - pivot_df = pivot_df.merge(stats_df, on="ProteinName", how="left") - pivot_df = pivot_df[["ProteinName", "log2FC", "p-value", "p-adj"] + all_samples + ["PeptideSequence"]] + # Ensure sample columns are numeric for downstream preprocessing/statistics. + pivot_df = df_cleaned.copy() + if sample_cols: + pivot_df[sample_cols] = pivot_df[sample_cols].apply(pd.to_numeric, errors='coerce') - # Build expression matrix (log2-transformed) - expr_df = pivot_df.set_index("ProteinName")[all_samples] - expr_df = expr_df.replace(0, np.nan) - expr_df = np.log2(expr_df + 1) - expr_df = expr_df.dropna() + protein_col = pivot_df.columns[0] + expr_df = pivot_df.set_index(protein_col)[sample_cols] + expr_df = expr_df.replace(0, np.nan) + expr_df = np.log2(expr_df + 1) + expr_df = expr_df.dropna() - return pivot_df, expr_df, group_map + return pivot_df, expr_df, group_map def get_abundance_data(workspace: Path) -> tuple | None: - """Wrapper that handles cache key (workspace + CSV mtime). + """Wrapper that handles cache key (workspace + CSV mtime + params mtime). Args: workspace: Path to the workspace directory @@ -324,4 +376,49 @@ def get_abundance_data(workspace: Path) -> tuple | None: return None csv_mtime = csv_files[0].stat().st_mtime - return load_abundance_data(str(workspace), csv_mtime) + + params_file = workflow_dir / "params.json" + params_mtime = params_file.stat().st_mtime if params_file.exists() else 0.0 + + return load_abundance_data(str(workspace), csv_mtime, params_mtime) + + +def get_id_column(workspace: Path, pivot_df: pd.DataFrame) -> str: + """Resolve the protein/row identifier column for the active analysis mode. + + LFQ reports always use "ProteinName"; TMT reports use whatever the + report's first column is actually named (e.g. "protein"). + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + return "ProteinName" if analysis_mode == "LFQ" else pivot_df.columns[0] + + +def get_sample_group_map(workspace: Path, pivot_df: pd.DataFrame, group_map: dict) -> dict: + """Normalize group_map into {actual_sample_column_name: group_name}. + + LFQ group_map keys are already clean sample names (optionally with a + ".mzML" suffix). TMT group_map keys are 0-based channel indices that must + be matched against the report's actual "sampleN[...]" column names. + """ + workflow_dir = get_workflow_dir(workspace) + analysis_mode = ParameterManager(workflow_dir, "TOPP Workflow").get_parameters_from_json().get("analysis-mode", "LFQ") + + if analysis_mode == "LFQ": + return { + k[:-5] if k.endswith(".mzML") else k: v + for k, v in group_map.items() + } + + actual_sample_names = pivot_df.columns.tolist() + norm_map = {} + for k, v in group_map.items(): + try: + sample_idx = int(k) + 1 + except (TypeError, ValueError): + continue + target_substring = f"sample{sample_idx}[" + real_full_name = next((name for name in actual_sample_names if target_substring in name), None) + if real_full_name: + norm_map[real_full_name] = v if v and v.strip() else "Unassigned" + return norm_map