demo_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43141e98",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import boto3\n",
    "from io import BytesIO\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import ray\n",
    "from ray import tune\n",
    "from ray.tune.callback import Callback\n",
    "from ray.tune.suggest.bohb import TuneBOHB\n",
    "from ray.tune.schedulers import HyperBandForBOHB\n",
    "from scipy.stats import loguniform, randint, uniform\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import tqdm\n",
    "from tqdm.notebook import trange, tqdm\n",
    "from tune_sklearn import TuneSearchCV\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import logging\n",
    "logging.disable(logging.INFO)\n",
    "logging.disable(logging.WARNING)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import xgboost as xgb\n",
    "xgb.set_config(verbosity=0)\n",
    "\n",
    "import os\n",
    "os.environ[\"TUNE_DISABLE_AUTO_CALLBACK_SYNCER\"] = \"1\"\n",
    "\n",
    "class TqdmCallback(Callback):\n",
    "    def setup(self,\n",
    "              stop = None,\n",
    "              num_samples = None,\n",
    "              total_num_samples = None,\n",
    "              **info):\n",
    "        self.pbar = tqdm(total=total_num_samples)\n",
    "\n",
    "    def on_trial_complete(self, **info):\n",
    "        self.pbar.update(1)\n",
    "\n",
    "    def on_experiment_end(self, **info):\n",
    "        self.pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85b724a5-9a31-414d-8330-ff885aa1c55b",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_cv_score(analysis):\n",
    "    df = analysis.results_df[[\"average_test_score\", \"timestamp\"]]\n",
    "    df[\"timestamp\"] = pd.to_datetime(df['timestamp'], unit='s')\n",
    "    df.set_index(\"timestamp\", inplace=True)\n",
    "    df.sort_index(inplace=True)\n",
    "    df[\"cummax_cv_score\"] = df[\"average_test_score\"].cummax()\n",
    "    df = df[~df.index.duplicated(keep=\"last\")]\n",
    "    df = df[\"cummax_cv_score\"].resample(\"1S\").bfill()\n",
    "    fig = px.line(df, y=\"cummax_cv_score\")\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e62e1a5",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "DATA_URL = \"https://ray-ci-higgs.s3.us-west-2.amazonaws.com/\" \\\n",
    "                      \"safe_driver.csv\"\n",
    "\n",
    "train_df = pd.read_csv(DATA_URL, dtype={'id': np.int32, 'target': np.int8})\n",
    "\n",
    "y = train_df['target'].values\n",
    "X = train_df.drop(['target', 'id'], axis=1)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2, random_state=1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ec29324-3c11-4f47-bc86-958e47d473b0",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def print_test_score(model, X_test, y_test):\n",
    "    y_pred = model.predict_proba(X_test)\n",
    "    roc_auc = roc_auc_score(y_test, y_pred[:,1])\n",
    "    print(\"**************** roc_auc score: {} ****************\".format(roc_auc))\n",
    "\n",
    "def train_model_and_print_test_score(model, X_train, y_train, X_test, y_test):\n",
    "    skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=1234)\n",
    "    run_cv = RandomizedSearchCV(model, param_distributions= {}, n_iter=1, scoring='roc_auc', n_jobs=-1, cv=skf.split(X_train,y_train), verbose=0, random_state=1001)\n",
    "    run_cv.fit(X_train, y_train)\n",
    "    print_test_score(run_cv.best_estimator_, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7763788b-8e67-41d6-98bb-74be189a26e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6760a164",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBClassifier(objective='binary:logistic', n_jobs=1, eval_metric='auc', random_state=1234, verbosity=0, use_label_encoder=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c5bdd76",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_model_and_print_test_score(model, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9cbfaa0-9ab2-4b84-97ed-35822d8207c9",
   "metadata": {},
   "source": [
    "## Now let's see how to do a distributed HPO using a Ray cluster!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efb5fcdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.init(address=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4724974f-4cd3-4f56-86ea-cfaa587db57b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.cluster_resources()['CPU']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b97f49",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "        \"max_depth\": randint(1, 5),\n",
    "        \"min_child_weight\": loguniform(0.001, 128),\n",
    "        \"subsample\": uniform(0.1, 1.0),\n",
    "        \"colsample_bylevel\": uniform(0.01, 1.0),\n",
    "        \"colsample_bytree\": uniform(0.01, 1.0),\n",
    "        \"reg_alpha\": loguniform(1 / 1024, 10.0),\n",
    "        \"reg_lambda\": loguniform(1 / 1024, 10.0),\n",
    "        \"scale_pos_weight\": [1, 26],\n",
    "}\n",
    "\n",
    "gs = RandomizedSearchCV(\n",
    "    model, \n",
    "    params,\n",
    "    cv=3,\n",
    "    n_iter=100,  # TODO: change to n_trials\n",
    "    scoring='roc_auc', \n",
    "    n_jobs=-1,  # TODO: change to 40verbose=0,\n",
    "#     early_stopping=True,\n",
    "#     max_iters=10,\n",
    "#     search_optimization=\"optuna\",\n",
    ")\n",
    "\n",
    "gs.fit(X_train, y_train) # , tune_params=dict(callbacks=[TqdmCallback()]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa7903d2-eed1-4d18-a027-ae8ab00699d9",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Results can be accessed similarly through `best_estimator_`. Tune also provides `ExperiementAnalysis` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb85a61c-6837-4e04-ac69-777e36c5f46d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bf2bc09-50ce-4ea7-86ab-c1767d75afd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}