Dataset integration

In this tutorial, we will integrate a differential centrifugation (DC) dataset with an organellar immunoprecipitation (OrgIP) dataset.

While DC is a more scalable technique, OrgIP provides significantly higher resolution. A practical approach to achieving high-resolution mapping is to strategically supplement DC data with targeted IP pulldowns for cellular compartments that are poorly resolved by centrifugation alone.

Although we merge two complete datasets in this example, real-world applications typically involve combining a DC dataset with only a small number of strategic IPs.

This integration process is streamlined using the grassp package.

import grassp as gr
import scanpy as sc
import anndata as ad
import numpy as np

/Users/mfrank/code/grassp/.venv-dev/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

Loading Data

# load DC dataset (hek cells)
dc = gr.ds.load_dataset("Frank_2026_HEK293")
dc

AnnData object with n_obs × n_vars = 8595 × 7
    obs: 'protein_name', 'gene_symbol', 'Protein IDs', 'Peptide counts (all)', 'Peptide counts (razor+unique)', 'Peptide counts (unique)', 'Protein names', 'Gene names', 'Number of proteins', 'Peptides', 'Razor + unique peptides', 'Unique peptides', 'Sequence coverage [%]', 'Unique + razor sequence coverage [%]', 'Unique sequence coverage [%]', 'Mol. weight [kDa]', 'Sequence length', 'Sequence lengths', 'Fraction average', 'Fraction 1', 'Fraction 2', 'Fraction 3', 'Q-value', 'Score', 'Intensity', 'iBAQ', 'MS/MS count', 'id', 'Peptide IDs', 'Peptide is razor', 'Mod. peptide IDs', 'Oxidation (M) site IDs', 'Oxidation (M) site positions', 'Length', 'Gene Names (primary)', 'marker_lilley', 'marker_christopher', 'marker_geladaki', 'marker_itzhak', 'marker_villaneuva', 'marker_hein2025', 'marker_hein2025_gt', 'n_samples_by_intensity', 'mean_intensity', 'log1p_mean_intensity', 'pct_dropout_by_intensity', 'total_intensity', 'log1p_total_intensity', 'n_imputed', 'leiden_0.5', 'leiden_0.8', 'leiden_1.0', 'leiden_2.0', 'leiden_3.0', 'leiden_5.0', 'leiden_8.0', 'leiden_12.0', 'leiden_15.0', 'leiden_15.0_CC_enrichment_consolidated', 'leiden_15.0_CC_enrichment_consolidated_Adjusted P-value Bonferroni', 'logEnrichmentPval', 'leiden_15.0_CC_enrichment_consolidated_filtered0.05', 'leiden_15.0_CC_enrichment_consolidated_filtered0.01', 'leiden_15.0_CC_enrichment_consolidated_filtered0.001', 'leiden_3.0_CC_enrichment', 'leiden_3.0_CC_enrichment_Adjusted P-value Bonferroni', 'leiden_15.0_CC_enrichment', 'leiden_15.0_CC_enrichment_Adjusted P-value Bonferroni', 'Topological domain', 'Transmembrane', 'Intramembrane', 'has_signal', 'has_transmem', 'has_intramem', 'has_topo_dom', 'mitocarta', 'mitocarta_evidence', 'mitocarta_subloc', 'mitocop', 'author_annotation', 'author_markers', 'unsupervised_annotation', 'harmonized_annotation_propagated', 'harmonized_annotation_propagated_probability'
    var: 'development_stage', 'tissue', 'sex', 'organism', 'disease', 'cell_line', 'cell_type', 'subcellular_enrichment', 'n_merged_samples', 'mean', 'std', 'enrichment_strategy', 'mass_spectrometry_method', 'cell_type_ontology_term_id', 'perturbation', 'disease_ontology_term_id', 'organism_ontology_term_id', 'sex_ontology_term_id', 'development_stage_ontology_term_id', 'tissue_ontology_term_id', 'cell_line_ontology_term_id', 'tissue_type', 'tissue_general_ontology_term_id', 'tissue_general', 'organ_ontology_term_id', 'organ', 'system_ontology_term_id', 'system'
    uns: 'RawInfo', 'harmonized_annotation_propagated_colors', 'has_intramem_colors', 'has_signal_colors', 'has_topo_dom_colors', 'has_transmem_colors', 'leiden_0.5', 'leiden_0.5_colors', 'leiden_0.8', 'leiden_0.8_colors', 'leiden_1.0', 'leiden_1.0_colors', 'leiden_12.0', 'leiden_12.0_colors', 'leiden_15.0', 'leiden_15.0_CC_enrichment_colors', 'leiden_15.0_CC_enrichment_consolidated_colors', 'leiden_15.0_CC_enrichment_consolidated_filtered0.001_colors', 'leiden_15.0_CC_enrichment_consolidated_filtered0.01_colors', 'leiden_15.0_CC_enrichment_consolidated_filtered0.05_colors', 'leiden_15.0_colors', 'leiden_2.0', 'leiden_2.0_colors', 'leiden_3.0', 'leiden_3.0_CC_enrichment_colors', 'leiden_3.0_colors', 'leiden_5.0', 'leiden_5.0_colors', 'leiden_8.0', 'leiden_8.0_colors', 'marker_christopher_colors', 'marker_geladaki_colors', 'marker_hein2025_colors', 'marker_hein2025_gt_colors', 'marker_itzhak_colors', 'marker_lilley_colors', 'marker_villaneuva_colors', 'mitocarta_colors', 'mitocop_colors', 'neighbors', 'pca', 'publication_authors', 'publication_date', 'publication_doi', 'publication_journal', 'publication_title', 'schema_reference', 'schema_version', 'serves_as_reference', 'title', 'umap', 'unsupervised_annotation_colors'
    obsm: 'X_pca', 'X_umap', 'X_umap3D', 'harmonized_annotation_propagated_one_hot_labels', 'harmonized_annotation_propagated_probabilities'
    varm: 'PCs'
    layers: 'MS_MS count', 'log_intensities', 'original_intensities', 'pvals', 'raw_intensities'
    obsp: 'connectivities', 'distances'

# OrgIP dataset (hek cells)
orgip = gr.ds.load_dataset("Hein_2025_HEK293_WT")
# This object ships the untagged (WT) control pulldowns and unscaled log fold
# changes. Drop the controls and z-score each bait, so that the ~70 IP columns
# and the 7 DC fractions contribute on a comparable scale when we concatenate.
orgip = orgip[:, ~orgip.var_names.str.endswith("-WT")].copy()
sc.pp.scale(orgip)
orgip

AnnData object with n_obs × n_vars = 8477 × 71
    obs: 'protein_name', 'gene_symbol', 'Protein IDs', 'author_gene_symbol', 'Majority protein IDs', 'Gene names', 'Gene_name_canonical', 'author_annotation', 'author_markers', 'marker_lilley', 'marker_christopher', 'marker_geladaki', 'marker_itzhak', 'marker_villaneuva', 'marker_hein2025', 'marker_hein2025_gt', 'n_samples_by_intensity', 'mean_intensity', 'pct_dropout_by_intensity', 'total_intensity', 'unsupervised_annotation', 'harmonized_annotation_propagated', 'harmonized_annotation_propagated_probability'
    var: 'development_stage', 'tissue', 'sex', 'organism', 'disease', 'cell_line', 'cell_type', 'subcellular_enrichment', 'fraction', 'fraction_order', 'biological_replicate', 'n_proteins_by_intensity', 'total_intensity', 'pct_dropout_by_intensity', 'enrichment_strategy', 'mass_spectrometry_method', 'cell_type_ontology_term_id', 'perturbation', 'disease_ontology_term_id', 'organism_ontology_term_id', 'sex_ontology_term_id', 'development_stage_ontology_term_id', 'tissue_ontology_term_id', 'cell_line_ontology_term_id', 'tissue_type', 'tissue_general_ontology_term_id', 'tissue_general', 'organ_ontology_term_id', 'organ', 'system_ontology_term_id', 'system', 'mean', 'std'
    uns: 'author_markers_colors', 'harmonized_annotation_propagated_colors', 'marker_christopher_colors', 'marker_geladaki_colors', 'marker_hein2025_colors', 'marker_hein2025_gt_colors', 'marker_itzhak_colors', 'marker_lilley_colors', 'marker_villaneuva_colors', 'neighbors', 'publication_authors', 'publication_date', 'publication_doi', 'publication_journal', 'publication_title', 'schema_reference', 'schema_version', 'serves_as_reference', 'title', 'unsupervised_annotation_colors'
    obsm: 'X_pca', 'X_umap', 'X_umap3D', 'harmonized_annotation_propagated_one_hot_labels', 'harmonized_annotation_propagated_probabilities'
    layers: 'pvals'
    obsp: 'connectivities', 'distances'

Plotting individual maps

Next, we use UMAP to visualize the spatial organization of the datasets. This helps us qualitatively assess how well the biological compartments are resolved before proceeding with downstream integration or analysis.

sc.pl.umap(orgip, color="marker_hein2025_gt")
sc.pl.umap(dc, color="marker_hein2025_gt")

Integration

To integrate the two datasets, we must identify a common gene or protein identifier to facilitate matching. While UniProt IDs or gene names are typical candidates, direct merging via UniProt IDs is challenging in this case because one dataset was searched against Swiss-Prot and the other against TrEMBL. To ensure consistency, we will first collapse all entries based on their gene names.

# aggregate proteins in the DC dataset
dc = dc[dc.obs["Gene names"].notna()]
dc_agg = gr.pp.aggregate_proteins(dc, grouping_columns="Gene names", agg_func=np.mean)
dc_agg.obs.head()

	protein_name	gene_symbol	Protein IDs	Peptide counts (all)	Peptide counts (razor+unique)	Peptide counts (unique)	Protein names	Gene names	Number of proteins	Peptides	...	mitocarta	mitocarta_evidence	mitocarta_subloc	mitocop	author_annotation	author_markers	unsupervised_annotation	harmonized_annotation_propagated	harmonized_annotation_propagated_probability	n_merged_proteins
AAAS	Aladin (Adracalin)	AAAS	Q9NRG9;Q9NRG9-2	24;20	24;20	24;20	Aladin	AAAS	2	24	...	NaN	NaN	NaN	NaN	Cell membrane	Cell membrane	Nucleus membrane	NaN	0.566277	1
AACS	Acetoacetyl-CoA synthetase (EC 6.2.1.16) (Acyl...	AACS	Q86V21;Q86V21-2;Q86V21-3	28;21;15	28;21;15	28;21;15	Acetoacetyl-CoA synthetase	AACS	3	28	...	NaN	NaN	NaN	NaN	unknown	unknown	Cytoplasm	Cytoplasm	1.0	1
AADAT	Kynurenine/alpha-aminoadipate aminotransferase...	AADAT	Q8N5Z0;Q8N5Z0-2	9;8	9;8	9;8	Kynurenine/alpha-aminoadipate aminotransferase...	AADAT	2	9	...	True	targetP signal, Rickettsial homolog, mito prot...	Matrix	True	Mitochondrion	Mitochondrion	Mitochondrion	Mitochondrion	1.0	1
AAED1	Peroxiredoxin-like 2C (AhpC/TSA antioxidant en...	PRXL2C	Q7RTV5	4	4	4	Thioredoxin-like protein AAED1	AAED1	1	4	...	NaN	NaN	NaN	NaN	Golgi apparatus	Golgi apparatus	Golgi apparatus	NaN	0.516824	1
AAGAB	Alpha- and gamma-adaptin-binding protein p34	AAGAB	Q6PD74;Q6PD74-2	10;6	10;6	10;6	Alpha- and gamma-adaptin-binding protein p34	AAGAB	2	10	...	NaN	NaN	NaN	NaN	Cytoplasm	Cytoplasm	Cytoplasm	Cytoplasm	0.918249	1

5 rows × 85 columns

# aggregate proteins in the OrgIP dataset
orgip_agg = gr.pp.aggregate_proteins(
    orgip, grouping_columns="Gene_name_canonical", agg_func=np.mean
)
orgip_agg.obs_names = orgip_agg.obs_names.str.upper()
orgip_agg.obs.head()

	protein_name	gene_symbol	Protein IDs	author_gene_symbol	Majority protein IDs	Gene names	Gene_name_canonical	author_annotation	author_markers	marker_lilley	...	marker_hein2025	marker_hein2025_gt	n_samples_by_intensity	mean_intensity	pct_dropout_by_intensity	total_intensity	unsupervised_annotation	harmonized_annotation_propagated	harmonized_annotation_propagated_probability	n_merged_proteins
A0A2R8Y3M9[P]	Group XIIA secretory phospholipase A2 (EC 3.1....	NaN	A0A2R8Y3M9	A0A2R8Y3M9[p]	A0A2R8Y3M9	NaN	A0A2R8Y3M9[p]	unclassified	NaN	NaN	...	NaN	NaN	83	0.016135	1.190476	1.355319	Centrosome	NaN	0.385025	1
A0A3B3ITR4[P]	Krev interaction trapped protein 1 (Leucine-ri...	NaN	A0A3B3ITR4;A0A0C4DG23;C9J718;C9JF32;C9JBN7;C9J...	A0A3B3ITR4[p]	A0A3B3ITR4	NaN	A0A3B3ITR4[p]	recycling_endosome	NaN	NaN	...	NaN	NaN	84	-0.65914	0.0	-55.367732	Cell membrane	Cell membrane	1.0	1
A0A5C2GRJ2[P]	IG c1334_heavy_IGHV3-73_IGHD6-6_IGHJ4	NaN	A0A5C2GRJ2	A0A5C2GRJ2[p]	A0A5C2GRJ2	NaN	A0A5C2GRJ2[p]	unclassified	NaN	NaN	...	NaN	NaN	82	-0.390949	2.380952	-32.839735	Nucleus	Nucleus	0.750001	1
A0A7D5Y1P9[P]	E1A binding protein p300	NaN	A0A7D5Y1P9	A0A7D5Y1P9[p]	A0A7D5Y1P9	NaN	A0A7D5Y1P9[p]	unclassified	NaN	NaN	...	NaN	NaN	84	0.00639	0.0	0.536731	Nucleus	Nucleus	0.740299	1
A0A384MDX9[P]	Far upstream element-binding protein 1	NaN	B4E0X8;A0A384MDX9	A0A384MDX9[p];B4E0X8[p]	B4E0X8;A0A384MDX9	NaN	A0A384MDX9[p]	nucleus	NaN	NaN	...	NaN	NaN	84	0.007124	0.0	0.598452	Nucleus	Nucleus	0.800081	1

5 rows × 24 columns

# combine the two datasets, keep only the common proteins,
combined = ad.concat(
    [dc_agg, orgip_agg],
    axis=1,  # This specifies a horizontal concatenation. We are keeping the proteins as rows and concatenating the experimental fractions or pulldowns from both datasets as columns.
    join="inner",
    merge="first",
    keys=["dc", "orgip"],
    label="dataset",
)
combined

AnnData object with n_obs × n_vars = 6997 × 78
    obs: 'protein_name', 'gene_symbol', 'Protein IDs', 'Peptide counts (all)', 'Peptide counts (razor+unique)', 'Peptide counts (unique)', 'Protein names', 'Gene names', 'Number of proteins', 'Peptides', 'Razor + unique peptides', 'Unique peptides', 'Sequence coverage [%]', 'Unique + razor sequence coverage [%]', 'Unique sequence coverage [%]', 'Mol. weight [kDa]', 'Sequence length', 'Sequence lengths', 'Fraction average', 'Fraction 1', 'Fraction 2', 'Fraction 3', 'Q-value', 'Score', 'Intensity', 'iBAQ', 'MS/MS count', 'id', 'Peptide IDs', 'Peptide is razor', 'Mod. peptide IDs', 'Oxidation (M) site IDs', 'Oxidation (M) site positions', 'Length', 'Gene Names (primary)', 'marker_lilley', 'marker_christopher', 'marker_geladaki', 'marker_itzhak', 'marker_villaneuva', 'marker_hein2025', 'marker_hein2025_gt', 'n_samples_by_intensity', 'mean_intensity', 'log1p_mean_intensity', 'pct_dropout_by_intensity', 'total_intensity', 'log1p_total_intensity', 'n_imputed', 'leiden_0.5', 'leiden_0.8', 'leiden_1.0', 'leiden_2.0', 'leiden_3.0', 'leiden_5.0', 'leiden_8.0', 'leiden_12.0', 'leiden_15.0', 'leiden_15.0_CC_enrichment_consolidated', 'leiden_15.0_CC_enrichment_consolidated_Adjusted P-value Bonferroni', 'logEnrichmentPval', 'leiden_15.0_CC_enrichment_consolidated_filtered0.05', 'leiden_15.0_CC_enrichment_consolidated_filtered0.01', 'leiden_15.0_CC_enrichment_consolidated_filtered0.001', 'leiden_3.0_CC_enrichment', 'leiden_3.0_CC_enrichment_Adjusted P-value Bonferroni', 'leiden_15.0_CC_enrichment', 'leiden_15.0_CC_enrichment_Adjusted P-value Bonferroni', 'Topological domain', 'Transmembrane', 'Intramembrane', 'has_signal', 'has_transmem', 'has_intramem', 'has_topo_dom', 'mitocarta', 'mitocarta_evidence', 'mitocarta_subloc', 'mitocop', 'author_annotation', 'author_markers', 'unsupervised_annotation', 'harmonized_annotation_propagated', 'harmonized_annotation_propagated_probability', 'n_merged_proteins', 'author_gene_symbol', 'Majority protein IDs', 'Gene_name_canonical'
    var: 'development_stage', 'tissue', 'sex', 'organism', 'disease', 'cell_line', 'cell_type', 'subcellular_enrichment', 'mean', 'std', 'enrichment_strategy', 'mass_spectrometry_method', 'cell_type_ontology_term_id', 'perturbation', 'disease_ontology_term_id', 'organism_ontology_term_id', 'sex_ontology_term_id', 'development_stage_ontology_term_id', 'tissue_ontology_term_id', 'cell_line_ontology_term_id', 'tissue_type', 'tissue_general_ontology_term_id', 'tissue_general', 'organ_ontology_term_id', 'organ', 'system_ontology_term_id', 'system', 'dataset'
    layers: 'pvals'

# compute UMAP embeddings
sc.pp.neighbors(combined, use_rep="X")
sc.tl.umap(combined, min_dist=0.1)

# plot UMAP of the integrated dataset
sc.pl.umap(combined, color="marker_hein2025_gt")