diff --git a/ml4h/explorations.py b/ml4h/explorations.py
index e14194f24..2d0800f1c 100644
--- a/ml4h/explorations.py
+++ b/ml4h/explorations.py
@@ -8,6 +8,7 @@
import logging
import operator
import datetime
+from scipy import stats
from functools import reduce
from itertools import combinations
from collections import defaultdict, Counter, OrderedDict
@@ -20,8 +21,10 @@
from sklearn.decomposition import PCA
from tensorflow.keras.models import Model
+
import matplotlib
matplotlib.use('Agg') # Need this to write images from the GSA servers. Order matters:
+import matplotlib.cm as cm
import matplotlib.pyplot as plt # First import matplotlib, then use Agg, then import plt
from ml4h.models.legacy_models import legacy_multimodal_multitask_model
@@ -33,10 +36,242 @@
from ml4h.defines import JOIN_CHAR, MRI_SEGMENTED_CHANNEL_MAP, CODING_VALUES_MISSING, CODING_VALUES_LESS_THAN_ONE
from ml4h.defines import TENSOR_EXT, IMAGE_EXT, ECG_CHAR_2_IDX, ECG_IDX_2_CHAR, PARTNERS_CHAR_2_IDX, PARTNERS_IDX_2_CHAR, PARTNERS_READ_TEXT
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.linear_model import LogisticRegression, LinearRegression, ElasticNet, Ridge, Lasso
+
CSV_EXT = '.tsv'
+def stratify_and_project_latent_space(stratify_column: str,stratify_thresh: float,stratify_std: float,latent_cols: List[str],
+ latent_df: pd.DataFrame,
+ normalize: bool = False,
+ train_ratio: int = 1.0):
+ """
+ Stratify data and project it to new latent space.
+ Args:
+ stratify_column (str): Name of the column used for stratification.
+ stratify_thresh (float): Threshold value for stratification.
+ stratify_std (float): Standard deviation value for stratification.
+ latent_cols (List[str]): List of column names for the latent space.
+ latent_df (pd.DataFrame): DataFrame containing the latent space data.
+ normalize (bool): Flag indicating whether to normalize the data. Default is False.
+ train_ratio (int): Ratio of training data to be used. Default is 1.0.
+
+ Returns:
+ Dict[str, Tuple[float,float,float]]
+ """
+ if train_ratio == 1.0:
+ train = latent_df
+ test = latent_df
+ else:
+ train = latent_df.sample(frac=train_ratio)
+ test = latent_df.drop(train.index)
+ hit = train.loc[train[stratify_column] >= stratify_thresh+(1*stratify_std)]
+ miss = train.loc[train[stratify_column] < stratify_thresh-(1*stratify_std)]
+ hit_np = hit[latent_cols].to_numpy()
+ miss_np = miss[latent_cols].to_numpy()
+ miss_mean_vector = np.mean(miss_np, axis=0)
+ hit_mean_vector = np.mean(hit_np, axis=0)
+ angle = angle_between(miss_mean_vector, hit_mean_vector)
+
+ hit_test = test.loc[test[stratify_column] >= stratify_thresh+(1*stratify_std)]
+ miss_test = test.loc[test[stratify_column] < stratify_thresh-(1*stratify_std)]
+
+ if normalize:
+ phenotype_vector = unit_vector(hit_mean_vector-miss_mean_vector)
+ hit_dots = [np.dot(phenotype_vector, unit_vector(v)) for v in hit_test[latent_cols].to_numpy()]
+ miss_dots = [np.dot(phenotype_vector, unit_vector(v)) for v in miss_test[latent_cols].to_numpy()]
+ else:
+ phenotype_vector = hit_mean_vector-miss_mean_vector
+ hit_dots = [np.dot(phenotype_vector, v) for v in hit_test[latent_cols].to_numpy()]
+ miss_dots = [np.dot(phenotype_vector, v) for v in miss_test[latent_cols].to_numpy()]
+ t2, p2 = stats.ttest_ind(hit_dots, miss_dots, equal_var = False)
+
+ return {f'{stratify_column}': (t2, p2, len(hit)) }
+
+
+
+def plot_nested_dictionary(all_scores: DefaultDict[str, DefaultDict[str, Tuple[float,float,float]]]) -> None:
+ """
+ Function to create a plot displaying T-statistics v/s Negative Log P-Value for each covariate.
+ Args:
+ all_scores (DefaultDict[str, DefaultDict[str, Tuple[float, float, float]]]): Nested dictionary containing the scores.
+ Returns:
+ None
+ """
+ n = 4
+ eps = 1e-300
+ for model in all_scores:
+ n = max(n, len(all_scores[model]))
+ cols = max(2, int(math.ceil(math.sqrt(n))))
+ rows = max(2, int(math.ceil(n / cols)))
+ fig, axes = plt.subplots(rows, cols, figsize=(cols * 4, rows * 3), sharex=True, dpi=300)
+ renest = defaultdict(dict)
+ errors = defaultdict(dict)
+ lens = {}
+ max_tstat = 0
+ max_pval = 0
+ for model in all_scores:
+ for metric in all_scores[model]:
+ renest[metric][model] = all_scores[model][metric][0]
+ errors[metric][model] = all_scores[model][metric][1]
+ lens[metric] = all_scores[model][metric][2]
+ max_tstat = max(abs(all_scores[model][metric][0]), max_tstat)
+ max_pval = max(-np.log10(all_scores[model][metric][1]+eps), max_pval)
+ for metric, ax in zip(renest, axes.ravel()):
+
+ models = [k for k,v in sorted(renest[metric].items(), key=lambda x: x[0].lower())]
+ tstats = [abs(v) for k,v in sorted(renest[metric].items(), key=lambda x: x[0].lower())]
+ pvalues = [-np.log10(v) if v > 1e-4800 else 500 for k,v in sorted(errors[metric].items(), key=lambda x: x[0].lower())]
+ y_pos = np.arange(len(models))
+ x = np.linspace(0, 1, int(max_pval))
+ plt.imshow(x[:, np.newaxis], cmap=cm.jet)
+ cb = plt.colorbar(ax=ax, ticks=[0, 1.0])
+ cb.set_label('Negative Log P-Value')
+ cb.ax.set_yticklabels(['0', f'{max_pval:0.0f}'])
+ ax.barh(y_pos, tstats, color=[cm.jet(p/max_pval) for p in pvalues], align='center')
+ ax.set_yticks(y_pos)
+ ax.set_yticklabels(models)
+ ax.invert_yaxis() # labels read top-to-bottom
+ ax.set_xlabel('T–Statistic')
+ ax.xaxis.set_tick_params(which='both', labelbottom=True)
+ ax.set_title(f'{metric}\n n={lens[metric]}')
+
+ plt.tight_layout()
+
+
+
+def angle_between(v1: np.ndarray, v2 : np.ndarray):
+ """ Returns the angle in radians between vectors 'v1' and 'v2'::
+ angle_between((1, 0, 0), (0, 1, 0))
+ 90
+ angle_between((1, 0, 0), (1, 0, 0))
+ 0.0
+ angle_between((1, 0, 0), (-1, 0, 0))
+ 180
+ """
+ v1_u = unit_vector(v1)
+ v2_u = unit_vector(v2)
+ return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0)) * 180 / 3.141592
+
+
+def unit_vector(vector : np.ndarray):
+ """ Returns the unit vector of the vector. """
+ return vector / np.linalg.norm(vector)
+
+
+def latent_space_dataframe(infer_hidden_tsv: str, explore_csv: str):
+ """
+ Read raw data from a CSV file and generate a representation of the data in a latent space.
+
+ Args:
+ infer_hidden_tsv (str): Path to the TSV file containing the inferred hidden representations.
+ explore_csv (str): Path to the CSV file containing the data to be explored.
+
+ Returns:
+ pandas.DataFrame: Dataframe representing the data in the latent space.
+
+ """
+ df = pd.read_csv(explore_csv)
+ df['sample_id'] = pd.to_numeric(df['fpath'], errors='coerce')
+ df2 = pd.read_csv(infer_hidden_tsv, sep='\t', engine='python')
+ df2['sample_id'] = pd.to_numeric(df2['sample_id'], errors='coerce')
+ latent_df = pd.merge(df, df2, on='sample_id', how='inner')
+ return latent_df
+
+
+def confounder_vector(labels: pd.Series, space: np.ndarray):
+ """
+ Compute the confounder vector based on labels and latent space.
+
+ Args:
+ labels (numpy.ndarray or list): The labels representing the dependent variable.
+ space (numpy.ndarray or list): The latent space representing the independent variable.
+
+ Returns:
+ cv and r2
+
+ """
+ clf = make_pipeline(StandardScaler(with_mean=True), Ridge(solver='lsqr'))
+ clf.fit(space, labels)
+ train_score = clf.score(space, labels)
+ return clf[-1].coef_/clf[0].scale_, train_score
+
+
+def confounder_matrix(adjust_cols: List[str], df: pd.DataFrame, space: np.ndarray):
+ """
+ Compute the confounder matrix based on specified columns, a dataframe, and a latent space.
+
+ Args:
+ adjust_cols (list): List of column names to adjust for as confounders.
+ df (pandas.DataFrame): The dataframe containing the data.
+ space (numpy.ndarray): The latent space representing the independent variable.
+
+ Returns:
+ computed confounder matrix and scores.
+
+ """
+ vectors = []
+ scores = {}
+ for col in adjust_cols:
+ cv, r2 = confounder_vector(df[col], space)
+ scores[col] = r2
+ vectors.append(cv)
+ return np.array(vectors), scores
+
+def iterative_subspace_removal(adjust_cols: List[str], latent_df: pd.DataFrame, latent_cols: List[str],
+ r2_thresh: float = 0.01, fit_pca: bool = False):
+ """
+ Perform iterative subspace removal based on specified columns, a latent dataframe,
+ and other parameters to remove confounder variables.
+
+ Args:
+ adjust_cols (List[str]): List of column names to adjust for as confounders.
+ latent_df (pd.DataFrame): The dataframe containing the latent data.
+ latent_cols (List[str]): List of column names representing the latent variables.
+ r2_thresh (float, optional): The threshold for the coefficient of determination (R-squared).
+ Default is 0.01.
+ fit_pca (bool, optional): Whether to fit Principal Component Analysis (PCA) on the latent data.
+ Default is False.
+
+ Returns:
+ pd.DataFrame: The latent dataframe after performing iterative subspace removal.
+ List: List of new columns
+
+ """
+ new_cols = latent_cols
+ new_adjust_cols = adjust_cols
+ space = latent_df[latent_cols].to_numpy()
+
+ if fit_pca:
+ pca = PCA()
+ pca.fit(space)
+ space = pca.transform(space)
+
+ iteration = 0
+ while len(new_adjust_cols) > 0 and space.shape[-1] > len(new_adjust_cols):
+ cfm, scores = confounder_matrix(new_adjust_cols, latent_df, space)
+ u, s, vt = np.linalg.svd(cfm, full_matrices=True)
+ nspace = np.matmul(space, vt[:, len(new_adjust_cols):])
+ new_cols=[f'new_latent_{iteration}_{i}' for i in range(nspace.shape[-1])]
+ df2 = pd.DataFrame(nspace, columns=new_cols, index=latent_df.index)
+ latent_df = pd.concat([latent_df, df2], axis=1)
+
+ iteration += 1
+ space = nspace
+
+ new_adjust_cols = [col for col, score in scores.items() if score > r2_thresh]
+ keep_cols = new_cols + [c for c in latent_df.columns if 'latent' not in c]
+ latent_df = latent_df[keep_cols]
+ r_scores= {k:round(v,4) for k,v in scores.items()}
+ print(f'Scores were {r_scores}, remaining columns are {new_adjust_cols}')
+ print(f'After iteration {iteration} Space shape is: {space.shape}')
+ return new_cols, latent_df
+
+
+
def predictions_to_pngs(
predictions: np.ndarray, tensor_maps_in: List[TensorMap], tensor_maps_out: List[TensorMap], data: Dict[str, np.ndarray],
labels: Dict[str, np.ndarray], paths: List[str], folder: str,
diff --git a/notebooks/latent_space_bias_detection.ipynb b/notebooks/latent_space_bias_detection.ipynb
new file mode 100644
index 000000000..ef4bf5a22
--- /dev/null
+++ b/notebooks/latent_space_bias_detection.ipynb
@@ -0,0 +1,944 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f22cd600",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "%load_ext autoreload\n",
+ "%autoreload 2"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "d5374fcd",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: nbformat in /usr/local/lib/python3.8/dist-packages (5.8.0)\n",
+ "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.8/dist-packages (from nbformat) (2.16.3)\n",
+ "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.8/dist-packages (from nbformat) (4.17.3)\n",
+ "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.8/dist-packages (from nbformat) (5.3.0)\n",
+ "Requirement already satisfied: traitlets>=5.1 in /usr/local/lib/python3.8/dist-packages (from nbformat) (5.9.0)\n",
+ "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema>=2.6->nbformat) (23.1.0)\n",
+ "Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema>=2.6->nbformat) (5.12.0)\n",
+ "Requirement already satisfied: pkgutil-resolve-name>=1.3.10 in /usr/local/lib/python3.8/dist-packages (from jsonschema>=2.6->nbformat) (1.3.10)\n",
+ "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.8/dist-packages (from jsonschema>=2.6->nbformat) (0.19.3)\n",
+ "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.8/dist-packages (from jupyter-core->nbformat) (3.5.0)\n",
+ "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/dist-packages (from importlib-resources>=1.4.0->jsonschema>=2.6->nbformat) (3.8.0)\n",
+ "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
+ "\u001b[0m"
+ ]
+ }
+ ],
+ "source": [
+ "# !pip install --upgrade nbformat\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "88039de4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%matplotlib inline\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "bd1c98ea",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: umap-learn[plot] in /usr/local/lib/python3.8/dist-packages (0.5.3)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (1.23.5)\n",
+ "Requirement already satisfied: scikit-learn>=0.22 in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (1.2.2)\n",
+ "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (1.9.1)\n",
+ "Requirement already satisfied: numba>=0.49 in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (0.56.4)\n",
+ "Requirement already satisfied: pynndescent>=0.5 in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (0.5.10)\n",
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (4.65.0)\n",
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (1.5.3)\n",
+ "Requirement already satisfied: matplotlib in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (3.7.1)\n",
+ "Collecting datashader (from umap-learn[plot])\n",
+ " Downloading datashader-0.15.0-py2.py3-none-any.whl (18.3 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.3/18.3 MB\u001b[0m \u001b[31m65.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: bokeh in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (3.1.0)\n",
+ "Collecting holoviews (from umap-learn[plot])\n",
+ " Downloading holoviews-1.16.0-py2.py3-none-any.whl (4.3 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.3/4.3 MB\u001b[0m \u001b[31m86.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
+ "\u001b[?25hCollecting colorcet (from umap-learn[plot])\n",
+ " Downloading colorcet-3.0.1-py2.py3-none-any.whl (1.7 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m77.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: seaborn in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (0.11.2)\n",
+ "Requirement already satisfied: scikit-image in /usr/local/lib/python3.8/dist-packages (from umap-learn[plot]) (0.20.0)\n",
+ "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn[plot]) (0.39.1)\n",
+ "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn[plot]) (62.3.2)\n",
+ "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.8/dist-packages (from numba>=0.49->umap-learn[plot]) (4.11.4)\n",
+ "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from pynndescent>=0.5->umap-learn[plot]) (1.2.0)\n",
+ "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=0.22->umap-learn[plot]) (3.1.0)\n",
+ "Requirement already satisfied: Jinja2>=2.9 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (3.1.2)\n",
+ "Requirement already satisfied: contourpy>=1 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (1.0.7)\n",
+ "Requirement already satisfied: packaging>=16.8 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (21.3)\n",
+ "Requirement already satisfied: pillow>=7.1.0 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (9.5.0)\n",
+ "Requirement already satisfied: PyYAML>=3.10 in /usr/lib/python3/dist-packages (from bokeh->umap-learn[plot]) (5.3.1)\n",
+ "Requirement already satisfied: tornado>=5.1 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (6.3.1)\n",
+ "Requirement already satisfied: xyzservices>=2021.09.1 in /usr/local/lib/python3.8/dist-packages (from bokeh->umap-learn[plot]) (2023.2.0)\n",
+ "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.8/dist-packages (from pandas->umap-learn[plot]) (2.8.2)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.8/dist-packages (from pandas->umap-learn[plot]) (2023.3)\n",
+ "Collecting pyct>=0.4.4 (from colorcet->umap-learn[plot])\n",
+ " Downloading pyct-0.5.0-py2.py3-none-any.whl (15 kB)\n",
+ "Collecting dask (from datashader->umap-learn[plot])\n",
+ " Downloading dask-2023.5.0-py3-none-any.whl (1.2 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting datashape (from datashader->umap-learn[plot])\n",
+ " Downloading datashape-0.5.2.tar.gz (76 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+ "\u001b[?25hCollecting param (from datashader->umap-learn[plot])\n",
+ " Downloading param-1.13.0-py2.py3-none-any.whl (87 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.3/87.3 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from datashader->umap-learn[plot]) (2.27.1)\n",
+ "Requirement already satisfied: toolz in /usr/local/lib/python3.8/dist-packages (from datashader->umap-learn[plot]) (0.12.0)\n",
+ "Collecting xarray (from datashader->umap-learn[plot])\n",
+ " Downloading xarray-2023.1.0-py3-none-any.whl (973 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m973.1/973.1 kB\u001b[0m \u001b[31m41.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting pyviz-comms>=0.7.4 (from holoviews->umap-learn[plot])\n",
+ " Downloading pyviz_comms-2.3.0-py2.py3-none-any.whl (43 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.8/43.8 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting panel>=0.13.1 (from holoviews->umap-learn[plot])\n",
+ " Downloading panel-1.0.4-py2.py3-none-any.whl (20.0 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.0/20.0 MB\u001b[0m \u001b[31m65.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.8/dist-packages (from matplotlib->umap-learn[plot]) (0.11.0)\n",
+ "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib->umap-learn[plot]) (4.39.3)\n",
+ "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib->umap-learn[plot]) (1.4.4)\n",
+ "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.8/dist-packages (from matplotlib->umap-learn[plot]) (3.0.9)\n",
+ "Requirement already satisfied: importlib-resources>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from matplotlib->umap-learn[plot]) (5.12.0)\n",
+ "Requirement already satisfied: networkx>=2.8 in /usr/local/lib/python3.8/dist-packages (from scikit-image->umap-learn[plot]) (3.1)\n",
+ "Requirement already satisfied: imageio>=2.4.1 in /usr/local/lib/python3.8/dist-packages (from scikit-image->umap-learn[plot]) (2.6.1)\n",
+ "Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.8/dist-packages (from scikit-image->umap-learn[plot]) (2023.4.12)\n",
+ "Requirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from scikit-image->umap-learn[plot]) (1.4.1)\n",
+ "Requirement already satisfied: lazy_loader>=0.1 in /usr/local/lib/python3.8/dist-packages (from scikit-image->umap-learn[plot]) (0.2)\n",
+ "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.8/dist-packages (from importlib-resources>=3.2.0->matplotlib->umap-learn[plot]) (3.8.0)\n",
+ "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/dist-packages (from Jinja2>=2.9->bokeh->umap-learn[plot]) (2.1.2)\n",
+ "Collecting bokeh (from umap-learn[plot])\n",
+ " Downloading bokeh-3.1.1-py3-none-any.whl (8.3 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.3/8.3 MB\u001b[0m \u001b[31m91.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: markdown in /usr/local/lib/python3.8/dist-packages (from panel>=0.13.1->holoviews->umap-learn[plot]) (3.3.7)\n",
+ "Collecting markdown-it-py (from panel>=0.13.1->holoviews->umap-learn[plot])\n",
+ " Downloading markdown_it_py-2.2.0-py3-none-any.whl (84 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.5/84.5 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting linkify-it-py (from panel>=0.13.1->holoviews->umap-learn[plot])\n",
+ " Downloading linkify_it_py-2.0.2-py3-none-any.whl (19 kB)\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Collecting mdit-py-plugins (from panel>=0.13.1->holoviews->umap-learn[plot])\n",
+ " Downloading mdit_py_plugins-0.3.5-py3-none-any.whl (52 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.1/52.1 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: bleach in /usr/local/lib/python3.8/dist-packages (from panel>=0.13.1->holoviews->umap-learn[plot]) (6.0.0)\n",
+ "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from panel>=0.13.1->holoviews->umap-learn[plot]) (4.5.0)\n",
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.8.1->pandas->umap-learn[plot]) (1.16.0)\n",
+ "Requirement already satisfied: click>=8.0 in /usr/local/lib/python3.8/dist-packages (from dask->datashader->umap-learn[plot]) (8.1.3)\n",
+ "Requirement already satisfied: cloudpickle>=1.5.0 in /usr/local/lib/python3.8/dist-packages (from dask->datashader->umap-learn[plot]) (2.2.1)\n",
+ "Collecting fsspec>=2021.09.0 (from dask->datashader->umap-learn[plot])\n",
+ " Downloading fsspec-2023.5.0-py3-none-any.whl (160 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m160.1/160.1 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting partd>=1.2.0 (from dask->datashader->umap-learn[plot])\n",
+ " Downloading partd-1.4.0-py3-none-any.whl (18 kB)\n",
+ "Collecting importlib-metadata (from numba>=0.49->umap-learn[plot])\n",
+ " Downloading importlib_metadata-6.6.0-py3-none-any.whl (22 kB)\n",
+ "Collecting multipledispatch>=0.4.7 (from datashape->datashader->umap-learn[plot])\n",
+ " Downloading multipledispatch-0.6.0-py3-none-any.whl (11 kB)\n",
+ "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->datashader->umap-learn[plot]) (1.26.9)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->datashader->umap-learn[plot]) (2022.5.18.1)\n",
+ "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.8/dist-packages (from requests->datashader->umap-learn[plot]) (2.0.12)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->datashader->umap-learn[plot]) (3.3)\n",
+ "Collecting locket (from partd>=1.2.0->dask->datashader->umap-learn[plot])\n",
+ " Downloading locket-1.0.0-py2.py3-none-any.whl (4.4 kB)\n",
+ "Requirement already satisfied: webencodings in /usr/local/lib/python3.8/dist-packages (from bleach->panel>=0.13.1->holoviews->umap-learn[plot]) (0.5.1)\n",
+ "Collecting uc-micro-py (from linkify-it-py->panel>=0.13.1->holoviews->umap-learn[plot])\n",
+ " Downloading uc_micro_py-1.0.2-py3-none-any.whl (6.2 kB)\n",
+ "Collecting mdurl~=0.1 (from markdown-it-py->panel>=0.13.1->holoviews->umap-learn[plot])\n",
+ " Downloading mdurl-0.1.2-py3-none-any.whl (10.0 kB)\n",
+ "Building wheels for collected packages: datashape\n",
+ " Building wheel for datashape (setup.py) ... \u001b[?25ldone\n",
+ "\u001b[?25h Created wheel for datashape: filename=datashape-0.5.2-py3-none-any.whl size=59421 sha256=6f67ef4aa51a3906c42cb3a17289eae2991cc0992e1a8f14e811687189db71e5\n",
+ " Stored in directory: /root/.cache/pip/wheels/6d/79/c4/c425774559165f472d32e5ef592ff9a71179abb31f05dbc98b\n",
+ "Successfully built datashape\n",
+ "Installing collected packages: uc-micro-py, param, multipledispatch, mdurl, locket, importlib-metadata, fsspec, pyviz-comms, pyct, partd, markdown-it-py, linkify-it-py, datashape, xarray, mdit-py-plugins, dask, colorcet, bokeh, panel, datashader, holoviews\n",
+ " Attempting uninstall: importlib-metadata\n",
+ " Found existing installation: importlib-metadata 4.11.4\n",
+ " Uninstalling importlib-metadata-4.11.4:\n",
+ " Successfully uninstalled importlib-metadata-4.11.4\n",
+ " Attempting uninstall: bokeh\n",
+ " Found existing installation: bokeh 3.1.0\n",
+ " Uninstalling bokeh-3.1.0:\n",
+ " Successfully uninstalled bokeh-3.1.0\n",
+ "Successfully installed bokeh-3.1.1 colorcet-3.0.1 dask-2023.5.0 datashader-0.15.0 datashape-0.5.2 fsspec-2023.5.0 holoviews-1.16.0 importlib-metadata-6.6.0 linkify-it-py-2.0.2 locket-1.0.0 markdown-it-py-2.2.0 mdit-py-plugins-0.3.5 mdurl-0.1.2 multipledispatch-0.6.0 panel-1.0.4 param-1.13.0 partd-1.4.0 pyct-0.5.0 pyviz-comms-2.3.0 uc-micro-py-1.0.2 xarray-2023.1.0\n",
+ "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
+ "\u001b[0m"
+ ]
+ }
+ ],
+ "source": [
+ "!pip install umap-learn[plot]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "id": "9669b72e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
+ "import umap.plot\n",
+ "import umap.umap_ as umap\n",
+ "from collections import defaultdict\n",
+ "import matplotlib.pyplot as plt\n",
+ "from ml4h.explorations import latent_space_dataframe,stratify_and_project_latent_space,plot_nested_dictionary,iterative_subspace_removal"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "edfbf94c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#code to copy files from google cloud\n",
+ "# !gsutil cp gs://fc-e5ea247f-18af-45e4-825a-71f0537e3cb7/hidden_ecg_rest_median_raw_10_dropout_pair_contrastive_lax_4ch_cycle_ecg_median_10_pretrained_256d_v2020_06_07.tsv latent_file.tsv\n",
+ " \n",
+ "\n",
+ "# !gsutil cp gs://fc-e5ea247f-18af-45e4-825a-71f0537e3cb7/tensors_all_union.csv tensors_all_union.csv\n",
+ " \n",
+ "# !gsutil cp gs://fc-e5ea247f-18af-45e4-825a-71f0537e3cb7/hidden_lax_4ch_heart_center_dropout_pair_contrastive_lax_4ch_cycle_ecg_median_10_pretrained_256d_v2020_06_07.tsv latent_file_for_comparison.tsv"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "6b50400a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Loading file path from local dir, gsutil cp was not working but you can try gsutil\n",
+ "ml4h_dir = os.path.dirname(os.getcwd())\n",
+ "latent_file = os.path.join(ml4h_dir,\"latent_data_test/latent_file.tsv\")\n",
+ "latent_file_for_comparison = os.path.join(ml4h_dir,\"latent_data_test/latent_file_for_comparison.tsv\")\n",
+ "label_file = os.path.join(ml4h_dir,\"latent_data_test/tensors_all_union.csv\") "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "3a369653",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# defining categorical and continuous covariates variables\n",
+ "categorical_covariates = ['Sex_Male_0_0',\n",
+ " 'atrial_fibrillation_or_flutter',\n",
+ " 'coronary_artery_disease',\n",
+ " 'diabetes_type_2',\n",
+ " 'hypercholesterolemia',\n",
+ " 'hypertension',\n",
+ " 'Sinus_bradycardia',\n",
+ " 'Marked_sinus_bradycardia',\n",
+ " 'Other_sinus_rhythm']\n",
+ "\n",
+ "continuous_covariates = ['age', 'bmi', 'RRInterval', 'QTInterval', 'LVM', 'RVEDV', 'PC1', 'PC2', 'PC3']\n",
+ "\n",
+ "all_scores = defaultdict(dict)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "5b90855f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['age', 'bmi', 'RRInterval', 'QTInterval', 'LVM', 'RVEDV', 'PC1', 'PC2', 'PC3']"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "continuous_covariates"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "c0136ab8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#renaming columns of latent file\n",
+ "col_rename = {f'22009_Genetic-principal-components_0_{i}': f'PC{i}' for i in range(1,41)}\n",
+ "col_rename['Genetic-sex_Male_0_0'] = 'sex'\n",
+ "col_rename['21003_Age-when-attended-assessment-centre_2_0'] = 'age'\n",
+ "col_rename['21001_Body-mass-index-BMI_2_0'] = 'bmi'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dcb8b975",
+ "metadata": {},
+ "source": [
+ "# ECG Inferences from contrasive paired model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "id": "5f12c555",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " PC1 | \n",
+ " PC2 | \n",
+ " PC3 | \n",
+ " PC4 | \n",
+ " PC5 | \n",
+ " PC6 | \n",
+ " PC7 | \n",
+ " PC8 | \n",
+ " PC9 | \n",
+ " PC10 | \n",
+ " ... | \n",
+ " latent_246 | \n",
+ " latent_247 | \n",
+ " latent_248 | \n",
+ " latent_249 | \n",
+ " latent_250 | \n",
+ " latent_251 | \n",
+ " latent_252 | \n",
+ " latent_253 | \n",
+ " latent_254 | \n",
+ " latent_255 | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " -13.040499 | \n",
+ " 5.09869 | \n",
+ " -0.941436 | \n",
+ " 1.10853 | \n",
+ " -1.113140 | \n",
+ " -0.551117 | \n",
+ " 0.108314 | \n",
+ " 0.384301 | \n",
+ " -1.836240 | \n",
+ " 0.841757 | \n",
+ " ... | \n",
+ " 0.357947 | \n",
+ " 0.572839 | \n",
+ " -0.151002 | \n",
+ " -0.141661 | \n",
+ " -0.005667 | \n",
+ " -0.119862 | \n",
+ " -0.850965 | \n",
+ " -0.756585 | \n",
+ " -0.147475 | \n",
+ " -0.007641 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " -16.290900 | \n",
+ " 1.73394 | \n",
+ " -3.384810 | \n",
+ " 2.69792 | \n",
+ " 0.750705 | \n",
+ " -1.378500 | \n",
+ " -3.941310 | \n",
+ " -2.645830 | \n",
+ " 3.677180 | \n",
+ " 1.559660 | \n",
+ " ... | \n",
+ " -0.120125 | \n",
+ " -0.580071 | \n",
+ " -0.149421 | \n",
+ " -0.140083 | \n",
+ " -0.004279 | \n",
+ " 0.092604 | \n",
+ " 0.947224 | \n",
+ " 0.245169 | \n",
+ " -0.145886 | \n",
+ " -0.006162 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " ... | \n",
+ " 1.308097 | \n",
+ " 1.045822 | \n",
+ " -0.201392 | \n",
+ " -0.191949 | \n",
+ " -0.049913 | \n",
+ " 0.289472 | \n",
+ " -0.622445 | \n",
+ " 0.984579 | \n",
+ " -0.198128 | \n",
+ " -0.054796 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " ... | \n",
+ " 0.501217 | \n",
+ " -0.023139 | \n",
+ " -0.136885 | \n",
+ " -0.127572 | \n",
+ " 0.006729 | \n",
+ " -0.030844 | \n",
+ " 0.685641 | \n",
+ " 0.009245 | \n",
+ " -0.133284 | \n",
+ " 0.005570 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " -11.620700 | \n",
+ " 5.15124 | \n",
+ " -1.085570 | \n",
+ " -2.52474 | \n",
+ " -2.436100 | \n",
+ " -1.334970 | \n",
+ " 1.850050 | \n",
+ " 1.236000 | \n",
+ " -0.165032 | \n",
+ " -2.820240 | \n",
+ " ... | \n",
+ " 0.368554 | \n",
+ " 0.684087 | \n",
+ " -0.157703 | \n",
+ " -0.148348 | \n",
+ " -0.011551 | \n",
+ " -0.086389 | \n",
+ " -0.356414 | \n",
+ " -0.425291 | \n",
+ " -0.154212 | \n",
+ " -0.013912 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
5 rows × 401 columns
\n",
+ "