example_BCWD.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Fetch dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "df_data, df_target = load_breast_cancer(return_X_y=True, as_frame=True)\n",
    "\n",
    "# Target contains index of target_names which are ['malignant', 'benign']\n",
    "# Flip the values so that target corresponds with malignancy\n",
    "df_target = (df_target*-1 + 1).astype(int)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Train AI network + 3. Full inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create ensemble of networks with fixed random states for reproducibility\n",
    "random_states = [1234, 2345, 3456, 4567]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notes:\n",
    "* no demographic information\n",
    "* no stratification\n",
    "* data split train/validation: 75% / 25%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import model_selection, ensemble\n",
    "import numpy as np\n",
    "\n",
    "# Create placeholder for AI output results\n",
    "df_predictions = pd.DataFrame()\n",
    "\n",
    "for model_id, random_state in enumerate(random_states):\n",
    "    print(f'model_id: {model_id}')\n",
    "    kf = model_selection.KFold(n_splits=4, shuffle=True, random_state=random_state)\n",
    "    rfc = ensemble.RandomForestClassifier(random_state=random_state)\n",
    "\n",
    "    df_prediction = df_target.copy(deep=True)\n",
    "    # df_prediction.name = 'model_{model_id}'\n",
    "\n",
    "    # For each fold: train, predict and store results\n",
    "    for idx, (train_index, validation_index) in enumerate(kf.split(df_data)):\n",
    "        rfc.fit(df_data.iloc[train_index], df_target.iloc[train_index])\n",
    "        prediction_class = rfc.predict(df_data.iloc[validation_index])\n",
    "        prediction_score = rfc.predict_proba(df_data.iloc[validation_index])[:,1]\n",
    "        \n",
    "        # Store prediction in df_output\n",
    "        df_prediction.iloc[validation_index] = prediction_score\n",
    "        \n",
    "        # Calculate and print sensitivity and specificity for fold\n",
    "        tn, fp, fn, tp = sklearn.metrics.confusion_matrix(df_target.iloc[validation_index], prediction_class, labels=[0, 1]).ravel()\n",
    "        sensitivity = tp / (tp + fn)\n",
    "        specificity = tn / (tn + fp)\n",
    "        \n",
    "        print(f'fold {idx} - sensitivity: {sensitivity} / specificity: {specificity}')\n",
    "\n",
    "    df_predictions[f'model_{model_id}'] = df_prediction\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Individual model sensitivity and specificity\n",
    "for model_id, c in enumerate(df_predictions.columns):\n",
    "    tn, fp, fn, tp = sklearn.metrics.confusion_matrix(df_target, df_predictions[c] > 0.5, labels=[0, 1]).ravel()\n",
    "    sensitivity = tp / (tp + fn)\n",
    "    specificity = tn / (tn + fp)\n",
    "    print(f'model {model_id} - sensitivity: {sensitivity} / specificity: {specificity}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensemble model sensitivity and specificity\n",
    "tn, fp, fn, tp = sklearn.metrics.confusion_matrix(df_target, df_predictions.mean(axis=1) > 0.5, labels=[0, 1]).ravel()\n",
    "sensitivity = tp / (tp + fn)\n",
    "specificity = tn / (tn + fp)\n",
    "print(f'Ensemble model - sensitivity: {sensitivity} / specificity: {specificity}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Aggregate and export data\n",
    "df_output = df_data.copy()\n",
    "df_output['gt'] = df_target\n",
    "for c in df_predictions.columns:\n",
    "    df_output[f'prediction_{c}'] = df_predictions[c]\n",
    "df_output['malignancy_score'] = df_predictions.mean(axis=1)\n",
    "df_output.index.name = 'record_id'\n",
    "\n",
    "df_output.to_csv('dataset_example_BCWD.csv')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Config file"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check config file ```settings/settings_data_slicing_pipeline_example_BCWD.py```\n",
    "\n",
    "##### Dataframe structure\n",
    "```\n",
    "'dataframe': {\n",
    "    'index': 'record_id',\n",
    "    # Additional columns to export from the original DataFrame\n",
    "    'export': [\n",
    "        'record_id',\n",
    "        'malignancy_score'\n",
    "    ],\n",
    "    'formatting': {\n",
    "        'binary_classification': {\n",
    "            'pred_label_input': 'malignancy_score',\n",
    "            'pred_label_output': 'malignancy_label_pred',\n",
    "            'gt_label_input': 'gt',\n",
    "            'gt_label_output': 'malignancy_label_gt',\n",
    "            'classification_threshold': 0.5,\n",
    "            'na_strict': False,\n",
    "            'na_fill': 'unknown',\n",
    "\n",
    "        },\n",
    "        'object_detection': {},\n",
    "        'segmentation': {},\n",
    "        'description_separator': '  '\n",
    "    }\n",
    "}\n",
    "```\n",
    "##### Slicing options\n",
    "```\n",
    "'slicing': {\n",
    "    'data_slicing_minimum_samples': 20,\n",
    "    # Format: 'column name as defined in dataframe': 'short column name to be used in the viewer'\n",
    "    'meta_data_fields_of_interest': {\n",
    "        'mean_radius': 'mean_radius',\n",
    "        'mean_texture': 'mean_texture',\n",
    "        'mean_perimeter': 'mean_perimeter',\n",
    "        'mean_area': 'mean_area',\n",
    "        'mean_smoothness': 'mean_smoothness',\n",
    "        'mean_compactness': 'mean_compactness',\n",
    "        'mean_concavity': 'mean_concavity',\n",
    "        'mean_concave_points': 'mean_concave_points',\n",
    "        'mean_symmetry': 'mean_symmetry',\n",
    "        'mean_fractal_dimension': 'mean_fractal_dimension'\n",
    "    }\n",
    "}\n",
    "```\n",
    "\n",
    "##### Additional metrics\n",
    "```\n",
    "'additional_metrics': {\n",
    "    MalignancyScore: {'probabilites_columns': 'malignancy_score'},\n",
    "    ConfidenceScore: {'probabilites_columns': list([f'prediction_model_{model_id}' for model_id in range(4)])},\n",
    "    OutlierScore: {'probabilites_columns': 'malignancy_score'}\n",
    "}\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Categorize values in dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _data_parsing_BCWD(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    import numpy as np\n",
    "    # fix incorrectly saved file if necessary\n",
    "    if 'level_0' in df.columns:\n",
    "        df = df.drop(columns='level_0')\n",
    "\n",
    "    # Rename column names to avoid spaces\n",
    "    df = df.rename(columns={c: c.replace(' ', '_') for c in df.columns})\n",
    "\n",
    "    # Categorize values in very_small, small, medium, large and very_large\n",
    "\n",
    "    metadata_columns = list(df.columns)\n",
    "    metadata_columns = list(c for c in metadata_columns if c not in ['gt', 'record_id', 'malignancy_score'])\n",
    "    metadata_columns = list(c for c in metadata_columns if 'prediction' not in c)\n",
    "\n",
    "    df_categorized = df.copy(deep=True)\n",
    "    categories = {'q00-20': [0, 20],\n",
    "                  'q20-40': [20, 40],\n",
    "                  'q40-60': [40, 60],\n",
    "                  'q60-80': [60, 80],\n",
    "                  'q80-100': [80, 100]}\n",
    "    for m in metadata_columns:\n",
    "        for c_label, c_percentiles in categories.items():\n",
    "            p = np.percentile(df[m], c_percentiles)\n",
    "            df_categorized.loc[(df[m] >= p[0]) & (df[m] < p[1]), m] = c_label\n",
    "    df = df_categorized\n",
    "    return df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Automated analysis pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Launch analysis pipeline\n",
    "from backend_launcher import run_data_slicing_experiment\n",
    "import pandas as pd\n",
    "import settings.settings_data_slicing_pipeline_example_BCWD_full as config_full\n",
    "import settings.settings_data_slicing_pipeline_example_BCWD_minimal as config_minimal\n",
    "\n",
    "# Keep the flag FULL_ANALYSIS set to False to prepare the dataframe only and use the analysis results provided with the repo.\n",
    "# Set to True to run a full analysis from scratch.\n",
    "FULL_ANALYSIS = False\n",
    "\n",
    "experiments_to_run = [\n",
    "    {\n",
    "        'config': config_minimal,\n",
    "        'df': pd.read_csv('dataset_example_BCWD.csv', low_memory=False),\n",
    "        'output_name': 'example_BCWD_minimal',\n",
    "        'optional_preprocessing_steps': [_data_parsing_BCWD],\n",
    "        'degrees': [0, 1, 2, 3],\n",
    "        'stop_after_df_preparation': not FULL_ANALYSIS\n",
    "    },\n",
    "    {\n",
    "        'config': config_full,\n",
    "        'df': pd.read_csv('dataset_example_BCWD.csv', low_memory=False),\n",
    "        'output_name': 'example_BCWD_full',\n",
    "        'optional_preprocessing_steps': [_data_parsing_BCWD],\n",
    "        'degrees': [0, 1, 2, 3],\n",
    "        'stop_after_df_preparation': not FULL_ANALYSIS\n",
    "    }\n",
    "]\n",
    "\n",
    "for experiment in experiments_to_run:\n",
    "    run_data_slicing_experiment(**experiment)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Dashboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -i dashboard.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.10 ('vivaldy_dashboard')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e850c2fd5fea9d09dd1bfa00a38957ce1995ecd0f7ccd96b7300b9fbad858fa3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}