example_notebooks/example_merge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pathlib\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "from idrstream.merge_CP_DP import merge_CP_DP_batch_data, save_merged_CP_DP_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cell_UUID</th>\n",
       "      <th>Location_Center_X</th>\n",
       "      <th>Location_Center_Y</th>\n",
       "      <th>Metadata_Plate</th>\n",
       "      <th>Metadata_Well</th>\n",
       "      <th>Metadata_Site</th>\n",
       "      <th>Metadata_Plate_Map_Name</th>\n",
       "      <th>Metadata_DNA</th>\n",
       "      <th>Metadata_Gene</th>\n",
       "      <th>Metadata_Gene_Replicate</th>\n",
       "      <th>...</th>\n",
       "      <th>DP__efficientnet_1270</th>\n",
       "      <th>DP__efficientnet_1271</th>\n",
       "      <th>DP__efficientnet_1272</th>\n",
       "      <th>DP__efficientnet_1273</th>\n",
       "      <th>DP__efficientnet_1274</th>\n",
       "      <th>DP__efficientnet_1275</th>\n",
       "      <th>DP__efficientnet_1276</th>\n",
       "      <th>DP__efficientnet_1277</th>\n",
       "      <th>DP__efficientnet_1278</th>\n",
       "      <th>DP__efficientnet_1279</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c53b0201-3e74-4d5b-8e13-6bc877b692b0</td>\n",
       "      <td>424</td>\n",
       "      <td>14</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>15_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_15</td>\n",
       "      <td>LT0001_02/LT0001_02_15_50.tif</td>\n",
       "      <td>negative control</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.070911</td>\n",
       "      <td>0.059313</td>\n",
       "      <td>-0.213210</td>\n",
       "      <td>0.288916</td>\n",
       "      <td>-0.163218</td>\n",
       "      <td>0.221167</td>\n",
       "      <td>-0.023267</td>\n",
       "      <td>-0.147827</td>\n",
       "      <td>-0.126854</td>\n",
       "      <td>1.112134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36fb5953-b7aa-4d03-94ed-18ab847de4e8</td>\n",
       "      <td>237</td>\n",
       "      <td>25</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>15_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_15</td>\n",
       "      <td>LT0001_02/LT0001_02_15_50.tif</td>\n",
       "      <td>negative control</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.080989</td>\n",
       "      <td>-0.171120</td>\n",
       "      <td>-0.041968</td>\n",
       "      <td>0.547571</td>\n",
       "      <td>-0.235733</td>\n",
       "      <td>-0.054545</td>\n",
       "      <td>-0.061243</td>\n",
       "      <td>-0.116886</td>\n",
       "      <td>-0.156546</td>\n",
       "      <td>0.639495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8bcd5101-9493-4c97-ab1c-d5dd205c2571</td>\n",
       "      <td>475</td>\n",
       "      <td>27</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>15_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_15</td>\n",
       "      <td>LT0001_02/LT0001_02_15_50.tif</td>\n",
       "      <td>negative control</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.103424</td>\n",
       "      <td>-0.208652</td>\n",
       "      <td>-0.216709</td>\n",
       "      <td>0.902428</td>\n",
       "      <td>-0.194517</td>\n",
       "      <td>0.149149</td>\n",
       "      <td>-0.028850</td>\n",
       "      <td>-0.110433</td>\n",
       "      <td>-0.175293</td>\n",
       "      <td>0.459885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3418a7aa-f5d2-44bf-adcf-324b8526e2ca</td>\n",
       "      <td>644</td>\n",
       "      <td>24</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>15_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_15</td>\n",
       "      <td>LT0001_02/LT0001_02_15_50.tif</td>\n",
       "      <td>negative control</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.018973</td>\n",
       "      <td>-0.038177</td>\n",
       "      <td>0.090443</td>\n",
       "      <td>-0.022167</td>\n",
       "      <td>-0.211883</td>\n",
       "      <td>0.373219</td>\n",
       "      <td>-0.042379</td>\n",
       "      <td>-0.212660</td>\n",
       "      <td>-0.044845</td>\n",
       "      <td>0.640951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>435493cf-239a-4ede-9038-89500ee41d57</td>\n",
       "      <td>1157</td>\n",
       "      <td>27</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>15_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_15</td>\n",
       "      <td>LT0001_02/LT0001_02_15_50.tif</td>\n",
       "      <td>negative control</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.145192</td>\n",
       "      <td>-0.087975</td>\n",
       "      <td>-0.200439</td>\n",
       "      <td>0.139166</td>\n",
       "      <td>-0.221564</td>\n",
       "      <td>0.236552</td>\n",
       "      <td>-0.099344</td>\n",
       "      <td>-0.200453</td>\n",
       "      <td>-0.128320</td>\n",
       "      <td>1.563802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>854</th>\n",
       "      <td>45d70ebd-8292-4ab1-90b4-cb1c8dd01093</td>\n",
       "      <td>582</td>\n",
       "      <td>997</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>4_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_4</td>\n",
       "      <td>LT0001_02/LT0001_02_4_50.tif</td>\n",
       "      <td>ENSG00000149503</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.047856</td>\n",
       "      <td>-0.194707</td>\n",
       "      <td>-0.178353</td>\n",
       "      <td>0.036657</td>\n",
       "      <td>-0.203256</td>\n",
       "      <td>0.046364</td>\n",
       "      <td>-0.025478</td>\n",
       "      <td>-0.155497</td>\n",
       "      <td>-0.162033</td>\n",
       "      <td>0.542753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>f4d49249-25bc-4cdb-8ce4-19ea3ba2f60f</td>\n",
       "      <td>211</td>\n",
       "      <td>993</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>4_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_4</td>\n",
       "      <td>LT0001_02/LT0001_02_4_50.tif</td>\n",
       "      <td>ENSG00000149503</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.025148</td>\n",
       "      <td>-0.157665</td>\n",
       "      <td>-0.185964</td>\n",
       "      <td>-0.153941</td>\n",
       "      <td>-0.061047</td>\n",
       "      <td>0.018391</td>\n",
       "      <td>-0.084140</td>\n",
       "      <td>-0.169552</td>\n",
       "      <td>-0.117730</td>\n",
       "      <td>0.342275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>507a40a4-70f7-49b4-a42c-e88fd82d7d3a</td>\n",
       "      <td>561</td>\n",
       "      <td>994</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>4_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_4</td>\n",
       "      <td>LT0001_02/LT0001_02_4_50.tif</td>\n",
       "      <td>ENSG00000149503</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.045312</td>\n",
       "      <td>-0.177917</td>\n",
       "      <td>-0.201178</td>\n",
       "      <td>-0.081857</td>\n",
       "      <td>-0.233622</td>\n",
       "      <td>0.202454</td>\n",
       "      <td>-0.030901</td>\n",
       "      <td>-0.116166</td>\n",
       "      <td>-0.093197</td>\n",
       "      <td>0.362397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>857</th>\n",
       "      <td>86b25c91-3203-4bb3-9a45-d0187abc7f65</td>\n",
       "      <td>1309</td>\n",
       "      <td>1002</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>4_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_4</td>\n",
       "      <td>LT0001_02/LT0001_02_4_50.tif</td>\n",
       "      <td>ENSG00000149503</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.039410</td>\n",
       "      <td>-0.161310</td>\n",
       "      <td>-0.120349</td>\n",
       "      <td>-0.112176</td>\n",
       "      <td>-0.171901</td>\n",
       "      <td>-0.044659</td>\n",
       "      <td>-0.045109</td>\n",
       "      <td>-0.182006</td>\n",
       "      <td>-0.067587</td>\n",
       "      <td>0.596588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>863e0034-a9b7-4f8f-823d-b5f9712a38fd</td>\n",
       "      <td>203</td>\n",
       "      <td>1010</td>\n",
       "      <td>LT0001_02</td>\n",
       "      <td>4_50</td>\n",
       "      <td>1</td>\n",
       "      <td>LT0001_02_4</td>\n",
       "      <td>LT0001_02/LT0001_02_4_50.tif</td>\n",
       "      <td>ENSG00000149503</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.027706</td>\n",
       "      <td>-0.161715</td>\n",
       "      <td>-0.176254</td>\n",
       "      <td>-0.156339</td>\n",
       "      <td>-0.128300</td>\n",
       "      <td>0.176655</td>\n",
       "      <td>-0.066756</td>\n",
       "      <td>-0.166198</td>\n",
       "      <td>-0.112588</td>\n",
       "      <td>0.356373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>859 rows × 1456 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Cell_UUID  Location_Center_X  \\\n",
       "0    c53b0201-3e74-4d5b-8e13-6bc877b692b0                424   \n",
       "1    36fb5953-b7aa-4d03-94ed-18ab847de4e8                237   \n",
       "2    8bcd5101-9493-4c97-ab1c-d5dd205c2571                475   \n",
       "3    3418a7aa-f5d2-44bf-adcf-324b8526e2ca                644   \n",
       "4    435493cf-239a-4ede-9038-89500ee41d57               1157   \n",
       "..                                    ...                ...   \n",
       "854  45d70ebd-8292-4ab1-90b4-cb1c8dd01093                582   \n",
       "855  f4d49249-25bc-4cdb-8ce4-19ea3ba2f60f                211   \n",
       "856  507a40a4-70f7-49b4-a42c-e88fd82d7d3a                561   \n",
       "857  86b25c91-3203-4bb3-9a45-d0187abc7f65               1309   \n",
       "858  863e0034-a9b7-4f8f-823d-b5f9712a38fd                203   \n",
       "\n",
       "     Location_Center_Y Metadata_Plate Metadata_Well  Metadata_Site  \\\n",
       "0                   14      LT0001_02         15_50              1   \n",
       "1                   25      LT0001_02         15_50              1   \n",
       "2                   27      LT0001_02         15_50              1   \n",
       "3                   24      LT0001_02         15_50              1   \n",
       "4                   27      LT0001_02         15_50              1   \n",
       "..                 ...            ...           ...            ...   \n",
       "854                997      LT0001_02          4_50              1   \n",
       "855                993      LT0001_02          4_50              1   \n",
       "856                994      LT0001_02          4_50              1   \n",
       "857               1002      LT0001_02          4_50              1   \n",
       "858               1010      LT0001_02          4_50              1   \n",
       "\n",
       "    Metadata_Plate_Map_Name                   Metadata_DNA     Metadata_Gene  \\\n",
       "0              LT0001_02_15  LT0001_02/LT0001_02_15_50.tif  negative control   \n",
       "1              LT0001_02_15  LT0001_02/LT0001_02_15_50.tif  negative control   \n",
       "2              LT0001_02_15  LT0001_02/LT0001_02_15_50.tif  negative control   \n",
       "3              LT0001_02_15  LT0001_02/LT0001_02_15_50.tif  negative control   \n",
       "4              LT0001_02_15  LT0001_02/LT0001_02_15_50.tif  negative control   \n",
       "..                      ...                            ...               ...   \n",
       "854             LT0001_02_4   LT0001_02/LT0001_02_4_50.tif   ENSG00000149503   \n",
       "855             LT0001_02_4   LT0001_02/LT0001_02_4_50.tif   ENSG00000149503   \n",
       "856             LT0001_02_4   LT0001_02/LT0001_02_4_50.tif   ENSG00000149503   \n",
       "857             LT0001_02_4   LT0001_02/LT0001_02_4_50.tif   ENSG00000149503   \n",
       "858             LT0001_02_4   LT0001_02/LT0001_02_4_50.tif   ENSG00000149503   \n",
       "\n",
       "     Metadata_Gene_Replicate  ...  DP__efficientnet_1270  \\\n",
       "0                          1  ...              -0.070911   \n",
       "1                          1  ...              -0.080989   \n",
       "2                          1  ...              -0.103424   \n",
       "3                          1  ...              -0.018973   \n",
       "4                          1  ...              -0.145192   \n",
       "..                       ...  ...                    ...   \n",
       "854                        1  ...              -0.047856   \n",
       "855                        1  ...              -0.025148   \n",
       "856                        1  ...              -0.045312   \n",
       "857                        1  ...              -0.039410   \n",
       "858                        1  ...              -0.027706   \n",
       "\n",
       "     DP__efficientnet_1271  DP__efficientnet_1272  DP__efficientnet_1273  \\\n",
       "0                 0.059313              -0.213210               0.288916   \n",
       "1                -0.171120              -0.041968               0.547571   \n",
       "2                -0.208652              -0.216709               0.902428   \n",
       "3                -0.038177               0.090443              -0.022167   \n",
       "4                -0.087975              -0.200439               0.139166   \n",
       "..                     ...                    ...                    ...   \n",
       "854              -0.194707              -0.178353               0.036657   \n",
       "855              -0.157665              -0.185964              -0.153941   \n",
       "856              -0.177917              -0.201178              -0.081857   \n",
       "857              -0.161310              -0.120349              -0.112176   \n",
       "858              -0.161715              -0.176254              -0.156339   \n",
       "\n",
       "     DP__efficientnet_1274  DP__efficientnet_1275  DP__efficientnet_1276  \\\n",
       "0                -0.163218               0.221167              -0.023267   \n",
       "1                -0.235733              -0.054545              -0.061243   \n",
       "2                -0.194517               0.149149              -0.028850   \n",
       "3                -0.211883               0.373219              -0.042379   \n",
       "4                -0.221564               0.236552              -0.099344   \n",
       "..                     ...                    ...                    ...   \n",
       "854              -0.203256               0.046364              -0.025478   \n",
       "855              -0.061047               0.018391              -0.084140   \n",
       "856              -0.233622               0.202454              -0.030901   \n",
       "857              -0.171901              -0.044659              -0.045109   \n",
       "858              -0.128300               0.176655              -0.066756   \n",
       "\n",
       "     DP__efficientnet_1277  DP__efficientnet_1278  DP__efficientnet_1279  \n",
       "0                -0.147827              -0.126854               1.112134  \n",
       "1                -0.116886              -0.156546               0.639495  \n",
       "2                -0.110433              -0.175293               0.459885  \n",
       "3                -0.212660              -0.044845               0.640951  \n",
       "4                -0.200453              -0.128320               1.563802  \n",
       "..                     ...                    ...                    ...  \n",
       "854              -0.155497              -0.162033               0.542753  \n",
       "855              -0.169552              -0.117730               0.342275  \n",
       "856              -0.116166              -0.093197               0.362397  \n",
       "857              -0.182006              -0.067587               0.596588  \n",
       "858              -0.166198              -0.112588               0.356373  \n",
       "\n",
       "[859 rows x 1456 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp_data_batch_0 = pd.read_csv(\n",
    "    \"../example_extracted_features/CP_features/batch_0.csv.gz\",\n",
    "    compression=\"gzip\",\n",
    "    index_col=0,\n",
    ")\n",
    "dp_data_batch_0 = pd.read_csv(\n",
    "    \"../example_extracted_features/DP_features/batch_0.csv.gz\",\n",
    "    compression=\"gzip\",\n",
    "    index_col=0,\n",
    ")\n",
    "\n",
    "merge_CP_DP_batch_data(cp_data_batch_0, dp_data_batch_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp_data_dir_path = pathlib.Path(\"../example_extracted_features/CP_features/\")\n",
    "dp_data_dir_path = pathlib.Path(\"../example_extracted_features/DP_features/\")\n",
    "merged_data_dir_path = pathlib.Path(\"../example_extracted_features/merged_features/\")\n",
    "\n",
    "save_merged_CP_DP_run(cp_data_dir_path, dp_data_dir_path, merged_data_dir_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('idrstream_cp')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "00511e853dd305362128087115a302b393aa6752b90db0accb1b4a560f5b1520"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}