Prepare-MIMIC-III-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use this script to process the MIMIC-III dataset extracted by Benchmarking_DL_MIMICIII and get the input files for the GRU-D models.\n",
    "\n",
    "Please set `benchmarking_root_folder` to the base directory of the __Benchmark_MIMIC_III__ project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmarking_root_folder = os.path.join('..', 'Benchmarking_DL_MIMICIII')   # This should be the [RD] (base directory) of the Benchmark_MIMIC_III project.\n",
    "grud_working_folder = '.'  # This should be the [WD] (working directory) of the GRU-D project.\n",
    "hours = 48\n",
    "benchmarking_data_folder = os.path.join(\n",
    "    benchmarking_root_folder, 'Data', 'admdata_99p',\n",
    "    '{}hrs_raw'.format(hours), 'series', 'mv'\n",
    ")\n",
    "print('The data files from the benchmarking codebase should be saved here:', benchmarking_data_folder)\n",
    "print('Files in this folder:', os.listdir(benchmarking_data_folder))\n",
    "\n",
    "grud_mimic3_data_folder = os.path.join(grud_working_folder, 'data', 'mimic3')\n",
    "if not os.path.exists(grud_mimic3_data_folder):\n",
    "    os.makedirs(grud_mimic3_data_folder)\n",
    "print('The data files will be converted and saved here:', grud_mimic3_data_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Convert normed-ep.npz to data.npz\n",
    "raw_data = np.load(os.path.join(benchmarking_data_folder, 'normed-ep.npz'))\n",
    "timestamp = raw_data['T_t']\n",
    "processed_data = {\n",
    "    'input': raw_data['X_t'],\n",
    "    'masking': raw_data['X_t_mask']\n",
    "}\n",
    "n = len(timestamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process input, masking, and timestamp, and keep only the first 48 hours\n",
    "for i in range(n):\n",
    "    if (i % 1000 == 0):\n",
    "        print('.', end='')\n",
    "    timestamp[i] = np.asarray(timestamp[i]) - timestamp[i][0]  # ensure the first timestamp is always 0\n",
    "    l = np.sum(timestamp[i] <= hours * 60 * 60)\n",
    "    timestamp[i] = timestamp[i][:l]\n",
    "    for k in processed_data.keys():\n",
    "        processed_data[k][i] = processed_data[k][i][:l]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process label (in-hospital mortality and ICD-9 categories)\n",
    "processed_label = {\n",
    "    'label_mortality': raw_data['y_mor'],\n",
    "    'label_icd9': raw_data['y_icd9']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez_compressed(os.path.join(grud_mimic3_data_folder, 'data.npz'), timestamp=timestamp, **processed_data, **processed_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process fold\n",
    "fold_data = np.load(os.path.join(benchmarking_data_folder, '5-folds.npz'))\n",
    "\n",
    "processed_fold = {\n",
    "    'fold_mortality': fold_data['folds_ep_mor'][0][0],\n",
    "    'fold_icd9': fold_data['folds_ep_icd9_multi'][0][0]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_stats = np.load(os.path.join(benchmarking_data_folder, 'normed-ep-stdized.npz'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process statistics\n",
    "# n_labeltype, 1, n_fold, (ts, nts), (mean, std)\n",
    "processed_stats = {\n",
    "    'mean_mortality': data_stats['folds_ep_mor'][0, 0, :, 0, 0],\n",
    "    'std_mortality': data_stats['folds_ep_mor'][0, 0, :, 0, 1],\n",
    "    'mean_icd9': data_stats['folds_ep_icd9_multi'][0, 0, :, 0, 0],\n",
    "    'std_icd9': data_stats['folds_ep_icd9_multi'][0, 0, :, 0, 1]\n",
    "}\n",
    "\n",
    "for k in processed_stats:\n",
    "    processed_stats[k] = np.repeat(processed_stats[k][:, np.newaxis], 3, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez_compressed(os.path.join(grud_mimic3_data_folder, 'fold.npz'), **processed_fold, **processed_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# size check\n",
    "data = np.load(os.path.join(grud_mimic3_data_folder, 'data.npz'))\n",
    "for k in data.keys():\n",
    "    print(k, data[k].shape)\n",
    "\n",
    "fold = np.load(os.path.join(grud_mimic3_data_folder, 'fold.npz'))\n",
    "for k in fold.keys():\n",
    "    print(k, fold[k].shape)\n",
    "    for f in fold[k]:\n",
    "        print('\\t', [x.shape for x in f])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Done!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}