TensorFlow/MultiClassReg/softmax.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 28, 28) (50000,)\n",
      "(10000, 28, 28) (10000,)\n",
      "(10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "os.chdir(\"D:\\\\Downloads\\\\\")\n",
    "from preprocessed_mnist import load_dataset\n",
    "\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_dataset()\n",
    "print(X_train.shape, y_train.shape)\n",
    "print(X_test.shape, y_test.shape)\n",
    "print(X_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x2bccbe403c8>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADOpJREFUeJzt3WGoXPWZx/Hfb932ja2gZNRgNTe3\nyLIibLoMYUmW1aXYpGsh9kVDA4YslE2jEVooolFChVC8Wbbt9sXacLsNTbFJU2hd88IkFSlmy12C\no0hNN2srybXNJiQ3WIx9VdSnL+5JuY13zkxmzpkzyfP9QJiZ85wz52HI756Z+c85f0eEAOTzF003\nAKAZhB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFJ/OcqdLVmyJCYmJka5SyCV2dlZnT9/3v2s\nO1T4ba+V9C1J10j6z4iYKlt/YmJCnU5nmF0CKNFut/ted+C3/bavkfQfkj4t6Q5JG2zfMejzARit\nYT7zr5T0RkSciIg/SPqhpHXVtAWgbsOE/xZJv13w+FSx7M/Y3my7Y7szNzc3xO4AVGmY8C/2pcIH\nzg+OiOmIaEdEu9VqDbE7AFUaJvynJN264PHHJJ0erh0AozJM+F+SdLvt5bY/LOnzkg5U0xaAug08\n1BcR79p+SNJhzQ/17Y6IX1bWGYBaDTXOHxHPSXquol4AjBA/7wWSIvxAUoQfSIrwA0kRfiApwg8k\nRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIP\nJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpoWbptT0r6R1J70l6NyLaVTQFoH5Dhb/wjxFxvoLn\nATBCvO0Hkho2/CHpp7Zftr25ioYAjMawb/tXR8Rp2zdKet72/0XEkYUrFH8UNkvSbbfdNuTuAFRl\nqCN/RJwubs9JekbSykXWmY6IdkS0W63WMLsDUKGBw2/7WtsfvXhf0qckHauqMQD1GuZt/02SnrF9\n8Xn2RsShSroCULuBwx8RJyT9TYW9AJW6cOFC19pTTz1Vuu2LL75YWj90qPw4t3bt2tL6wYMHS+uj\nwFAfkBThB5Ii/EBShB9IivADSRF+IKkqzuoDBjIzM1NaP3LkSGl92OG4Ok1MTDS2735x5AeSIvxA\nUoQfSIrwA0kRfiApwg8kRfiBpBjnR6my02Kl3qfGTk9Pd62dPHlyoJ6q0OuU2+3bt5fWV61aVWU7\njeDIDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJMc6PUo888khpfdeuXQM/95YtW0rrGzduHPi5patj\nLL5OHPmBpAg/kBThB5Ii/EBShB9IivADSRF+IKme4/y2d0v6jKRzEXFnsewGSfslTUialbQ+In5X\nX5uoy9TUVGn98OHDpfVeY/UPP/xw19rk5GTptqhXP0f+70m69MoHj0p6ISJul/RC8RjAFaRn+CPi\niKS3Llm8TtKe4v4eSfdV3BeAmg36mf+miDgjScXtjdW1BGAUav/Cz/Zm2x3bnbm5ubp3B6BPg4b/\nrO2lklTcnuu2YkRMR0Q7ItqtVmvA3QGo2qDhPyBpU3F/k6Rnq2kHwKj0DL/tfZL+R9Jf2T5l+wuS\npiTdY/vXku4pHgO4gvQc54+IDV1Kn6y4F9RgZmamtL5t27bSeq/r2+/cubO0ft1115XW0Rx+4Qck\nRfiBpAg/kBThB5Ii/EBShB9Iikt3X+V27Ngx1PZ33XVXaf3YsWOl9ZtvvrlrjVN6m8WRH0iK8ANJ\nEX4gKcIPJEX4gaQIP5AU4QeSYpz/KvDAAw90rR06dKh0216n7C5btqy0fv/995fW16xZ07XG6cDN\n4sgPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kxzj8GTpw4UVrfunVrab1sLH/v3r2l2957772l9V5j\n7W+++WZpvezS4L3G+VEvjvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kFTPcX7buyV9RtK5iLizWPaE\npH+RNFes9lhEPFdXk1e7o0ePltZ7nZNfNpa/YUO3GdaRXT9H/u9JWuyKD9+MiBXFP4IPXGF6hj8i\njkh6awS9ABihYT7zP2T7F7Z3276+so4AjMSg4f+2pI9LWiHpjKSvd1vR9mbbHdudubm5bqsBGLGB\nwh8RZyPivYh4X9J3JK0sWXc6ItoR0W61WoP2CaBiA4Xf9tIFDz8rqXyqVgBjp5+hvn2S7pa0xPYp\nSV+VdLftFZJC0qykL9bYI4AaOCJGtrN2ux2dTmdk+7taXLhwobTe5PXtJycnS+snT57sWnv77bdL\nt+W6/Zev3W6r0+m4n3X5hR+QFOEHkiL8QFKEH0iK8ANJEX4gKS7dfQVocshr3759pfWyoTxJevLJ\nJ7vWGMprFkd+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iKcf7CzMxMaX3VqlUj6mS0eo3jP/7446X1\n5cuXl9YffPDBy+4Jo8GRH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSYpy/sHr16tL62rWLTVQ8b/v2\n7aXb1v0bgampqa61bdu2DfXcW7ZsKa3v3LmztM45++OLIz+QFOEHkiL8QFKEH0iK8ANJEX4gKcIP\nJNVznN/2rZK+L+lmSe9Lmo6Ib9m+QdJ+SROSZiWtj4jf1ddqvXqdl37o0KGutddff7102zVr1pTW\nDx8+XFrvdW38MmW/T5Ck/fv3l9YZp7969XPkf1fSVyLiryX9naSttu+Q9KikFyLidkkvFI8BXCF6\nhj8izkTEK8X9dyQdl3SLpHWS9hSr7ZF0X11NAqjeZX3mtz0h6ROSjkq6KSLOSPN/ICTdWHVzAOrT\nd/htf0TSjyV9OSIuXMZ2m213bHfm5uYG6RFADfoKv+0PaT74P4iInxSLz9peWtSXSjq32LYRMR0R\n7Yhot1qtKnoGUIGe4bdtSd+VdDwivrGgdEDSpuL+JknPVt8egLr0c0rvakkbJb1m+9Vi2WOSpiT9\nyPYXJP1G0ufqaXE0nn766dL6jh07utbKhgEladeuXaX1XqfNLlu2rLS+fv36rrXJycnSbZFXz/BH\nxM8luUv5k9W2A2BU+IUfkBThB5Ii/EBShB9IivADSRF+ICku3V3odXntgwcPjqgTYDQ48gNJEX4g\nKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+\nICnCDyRF+IGkCD+QFOEHkiL8QFI9w2/7Vts/s33c9i9tf6lY/oTt/7f9avHvn+pvF0BV+pm0411J\nX4mIV2x/VNLLtp8vat+MiH+rrz0AdekZ/og4I+lMcf8d28cl3VJ3YwDqdVmf+W1PSPqEpKPFoods\n/8L2btvXd9lms+2O7c7c3NxQzQKoTt/ht/0RST+W9OWIuCDp25I+LmmF5t8ZfH2x7SJiOiLaEdFu\ntVoVtAygCn2F3/aHNB/8H0TETyQpIs5GxHsR8b6k70haWV+bAKrWz7f9lvRdSccj4hsLli9dsNpn\nJR2rvj0Adenn2/7VkjZKes32q8WyxyRtsL1CUkialfTFWjoEUIt+vu3/uSQvUnqu+nYAjAq/8AOS\nIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyTliBjdzuw5SW8u\nWLRE0vmRNXB5xrW3ce1LordBVdnbsojo63p5Iw3/B3ZudyKi3VgDJca1t3HtS6K3QTXVG2/7gaQI\nP5BU0+Gfbnj/Zca1t3HtS6K3QTXSW6Of+QE0p+kjP4CGNBJ+22ttv277DduPNtFDN7Znbb9WzDzc\nabiX3bbP2T62YNkNtp+3/evidtFp0hrqbSxmbi6ZWbrR127cZrwe+dt+29dI+pWkeySdkvSSpA0R\n8b8jbaQL27OS2hHR+Jiw7X+Q9HtJ34+IO4tl/yrprYiYKv5wXh8Rj4xJb09I+n3TMzcXE8osXTiz\ntKT7JP2zGnztSvparwZetyaO/CslvRERJyLiD5J+KGldA32MvYg4IumtSxavk7SnuL9H8/95Rq5L\nb2MhIs5ExCvF/XckXZxZutHXrqSvRjQR/lsk/XbB41Marym/Q9JPbb9se3PTzSzipmLa9IvTp9/Y\ncD+X6jlz8yhdMrP02Lx2g8x4XbUmwr/Y7D/jNOSwOiL+VtKnJW0t3t6iP33N3Dwqi8wsPRYGnfG6\nak2E/5SkWxc8/pik0w30saiIOF3cnpP0jMZv9uGzFydJLW7PNdzPn4zTzM2LzSytMXjtxmnG6ybC\n/5Kk220vt/1hSZ+XdKCBPj7A9rXFFzGyfa2kT2n8Zh8+IGlTcX+TpGcb7OXPjMvMzd1mllbDr924\nzXjdyI98iqGMf5d0jaTdEfG1kTexCNuTmj/aS/OTmO5tsjfb+yTdrfmzvs5K+qqk/5L0I0m3SfqN\npM9FxMi/eOvS292af+v6p5mbL37GHnFvfy/pvyW9Jun9YvFjmv983dhrV9LXBjXwuvELPyApfuEH\nJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpPwI9Ja5WGlRTUAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2bccaa5a5f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.imshow(X_train[35], cmap=\"Greys\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic regression\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = tf.Session()\n",
    "\n",
    "# Reshape features to flat format\n",
    "X_train_flat = s.run(tf.reshape(X_train, [X_train.shape[0],-1]))\n",
    "X_test_flat  = s.run(tf.reshape(X_test, [X_test.shape[0],-1]))\n",
    "X_val_flat   = s.run(tf.reshape(X_val, [X_val.shape[0],-1]))\n",
    "\n",
    "# Categorical labels to binaries\n",
    "y_train_oh = s.run(tf.one_hot(y_train, 10))\n",
    "y_test_oh  = s.run(tf.one_hot(y_test, 10))\n",
    "y_val_oh   = s.run(tf.one_hot(y_val, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'Placeholder:0' shape=(?, 784) dtype=float32>,\n",
       " <tf.Tensor 'Placeholder_1:0' shape=(?, 10) dtype=float32>,\n",
       " <tf.Variable 'weights:0' shape=(784, 10) dtype=float32_ref>,\n",
       " <tf.Variable 'biases:0' shape=(10,) dtype=float32_ref>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model parameters - weights and bias\n",
    "nuniq = len(np.unique(y_train))\n",
    "weights = tf.Variable(tf.random_normal([X_train_flat.shape[1],nuniq], stddev=0.35),\n",
    "                      name=\"weights\") \n",
    "\n",
    "b = tf.Variable(tf.zeros(nuniq), dtype='float32', name=\"biases\")\n",
    "\n",
    "# Placeholders for the input data\n",
    "input_X = tf.placeholder('float32', shape=(None,X_train_flat.shape[1]))\n",
    "input_y = tf.placeholder('float32', shape=(None, nuniq))\n",
    "input_X, input_y, weights, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "#predicted_y =  tf.nn.softmax(tf.matmul(input_X, weights)+b)\n",
    "predicted_y =  tf.matmul(input_X, weights)+b\n",
    "\n",
    "# Loss. Should be a scalar number - average loss over all the objects\n",
    "#loss = tf.reduce_mean(-tf.reduce_sum(tf.log(predicted_y+1e-07)*input_y, reduction_indices=[1]))\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=input_y, logits=predicted_y))\n",
    "\n",
    "# See above for an example. tf.train.*Optimizer\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(\n",
    "    loss, var_list=(weights,b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss at iter 0:3.7550\n",
      "train auc: 0.549589412678\n",
      "test auc: 0.560197420286\n",
      "loss at iter 1:3.1837\n",
      "train auc: 0.602448621057\n",
      "test auc: 0.614358211862\n",
      "loss at iter 2:2.7913\n",
      "train auc: 0.651829728162\n",
      "test auc: 0.664631689652\n",
      "loss at iter 3:2.4778\n",
      "train auc: 0.694817828585\n",
      "test auc: 0.708154272824\n",
      "loss at iter 4:2.2259\n",
      "train auc: 0.730444319259\n",
      "test auc: 0.743931209001\n",
      "loss at iter 5:2.0218\n",
      "train auc: 0.759753115082\n",
      "test auc: 0.773140225809\n",
      "loss at iter 6:1.8546\n",
      "train auc: 0.783683692632\n",
      "test auc: 0.796789951741\n",
      "loss at iter 7:1.7161\n",
      "train auc: 0.803341899379\n",
      "test auc: 0.816117544035\n",
      "loss at iter 8:1.6000\n",
      "train auc: 0.819573470775\n",
      "test auc: 0.831974330392\n",
      "loss at iter 9:1.5019\n",
      "train auc: 0.833107781835\n",
      "test auc: 0.84516366013\n",
      "loss at iter 10:1.4181\n",
      "train auc: 0.844490629034\n",
      "test auc: 0.856199483986\n",
      "loss at iter 11:1.3458\n",
      "train auc: 0.85416814746\n",
      "test auc: 0.865527487545\n",
      "loss at iter 12:1.2831\n",
      "train auc: 0.862467652407\n",
      "test auc: 0.87349330184\n",
      "loss at iter 13:1.2281\n",
      "train auc: 0.869644776222\n",
      "test auc: 0.880339328059\n",
      "loss at iter 14:1.1796\n",
      "train auc: 0.875907621207\n",
      "test auc: 0.886299932531\n",
      "loss at iter 15:1.1366\n",
      "train auc: 0.881408915657\n",
      "test auc: 0.89150696796\n",
      "loss at iter 16:1.0981\n",
      "train auc: 0.886280535494\n",
      "test auc: 0.896110530447\n",
      "loss at iter 17:1.0635\n",
      "train auc: 0.890623065659\n",
      "test auc: 0.9002016562\n",
      "loss at iter 18:1.0322\n",
      "train auc: 0.894513144894\n",
      "test auc: 0.903849724255\n",
      "loss at iter 19:1.0038\n",
      "train auc: 0.898018319286\n",
      "test auc: 0.907127654095\n",
      "loss at iter 20:0.9779\n",
      "train auc: 0.901190130367\n",
      "test auc: 0.910095531961\n",
      "loss at iter 21:0.9541\n",
      "train auc: 0.90407806513\n",
      "test auc: 0.91279055696\n",
      "loss at iter 22:0.9323\n",
      "train auc: 0.906715961025\n",
      "test auc: 0.915243781956\n",
      "loss at iter 23:0.9121\n",
      "train auc: 0.909136518798\n",
      "test auc: 0.917495502165\n",
      "loss at iter 24:0.8934\n",
      "train auc: 0.911361612075\n",
      "test auc: 0.9195592929\n",
      "loss at iter 25:0.8761\n",
      "train auc: 0.913419926015\n",
      "test auc: 0.921480064743\n",
      "loss at iter 26:0.8599\n",
      "train auc: 0.91532615332\n",
      "test auc: 0.92324607651\n",
      "loss at iter 27:0.8448\n",
      "train auc: 0.917097391892\n",
      "test auc: 0.924889383606\n",
      "loss at iter 28:0.8306\n",
      "train auc: 0.918748578759\n",
      "test auc: 0.926416033509\n",
      "loss at iter 29:0.8173\n",
      "train auc: 0.920289946313\n",
      "test auc: 0.927839458129\n",
      "loss at iter 30:0.8048\n",
      "train auc: 0.921733893348\n",
      "test auc: 0.929168934415\n",
      "loss at iter 31:0.7930\n",
      "train auc: 0.923088917427\n",
      "test auc: 0.93041988138\n",
      "loss at iter 32:0.7818\n",
      "train auc: 0.924363440134\n",
      "test auc: 0.931590123709\n",
      "loss at iter 33:0.7713\n",
      "train auc: 0.925563842508\n",
      "test auc: 0.93269697527\n",
      "loss at iter 34:0.7613\n",
      "train auc: 0.926696983619\n",
      "test auc: 0.933739085659\n",
      "loss at iter 35:0.7518\n",
      "train auc: 0.927768215851\n",
      "test auc: 0.934724314559\n",
      "loss at iter 36:0.7427\n",
      "train auc: 0.928784276755\n",
      "test auc: 0.935654623062\n",
      "loss at iter 37:0.7341\n",
      "train auc: 0.929747580739\n",
      "test auc: 0.936535066427\n",
      "loss at iter 38:0.7259\n",
      "train auc: 0.930664349688\n",
      "test auc: 0.937368012401\n",
      "loss at iter 39:0.7181\n",
      "train auc: 0.931534722156\n",
      "test auc: 0.93815787857\n",
      "loss at iter 40:0.7105\n",
      "train auc: 0.932364994118\n",
      "test auc: 0.938909649588\n",
      "loss at iter 41:0.7034\n",
      "train auc: 0.933157551139\n",
      "test auc: 0.939629956556\n",
      "loss at iter 42:0.6965\n",
      "train auc: 0.933912912133\n",
      "test auc: 0.940314014985\n",
      "loss at iter 43:0.6899\n",
      "train auc: 0.934635640408\n",
      "test auc: 0.940969409811\n",
      "loss at iter 44:0.6835\n",
      "train auc: 0.935327674198\n",
      "test auc: 0.941597670375\n",
      "loss at iter 45:0.6774\n",
      "train auc: 0.935991804352\n",
      "test auc: 0.942197317654\n",
      "loss at iter 46:0.6715\n",
      "train auc: 0.936627830658\n",
      "test auc: 0.942769268339\n",
      "loss at iter 47:0.6659\n",
      "train auc: 0.937237850085\n",
      "test auc: 0.943316634788\n",
      "loss at iter 48:0.6604\n",
      "train auc: 0.937825478785\n",
      "test auc: 0.94384665059\n",
      "loss at iter 49:0.6551\n",
      "train auc: 0.938390847912\n",
      "test auc: 0.944357046261\n",
      "loss at iter 50:0.6501\n",
      "train auc: 0.938933774723\n",
      "test auc: 0.944841176999\n",
      "loss at iter 51:0.6451\n",
      "train auc: 0.939457105971\n",
      "test auc: 0.945308711136\n",
      "loss at iter 52:0.6404\n",
      "train auc: 0.939961871727\n",
      "test auc: 0.945755182443\n",
      "loss at iter 53:0.6358\n",
      "train auc: 0.940448710679\n",
      "test auc: 0.946186528968\n",
      "loss at iter 54:0.6313\n",
      "train auc: 0.940918588558\n",
      "test auc: 0.946599079804\n",
      "loss at iter 55:0.6270\n",
      "train auc: 0.941371962998\n",
      "test auc: 0.947006008787\n",
      "loss at iter 56:0.6228\n",
      "train auc: 0.941810939422\n",
      "test auc: 0.947392088226\n",
      "loss at iter 57:0.6188\n",
      "train auc: 0.942235233135\n",
      "test auc: 0.947767352411\n",
      "loss at iter 58:0.6148\n",
      "train auc: 0.942645685538\n",
      "test auc: 0.948129612488\n",
      "loss at iter 59:0.6110\n",
      "train auc: 0.94304297867\n",
      "test auc: 0.948483277611\n",
      "loss at iter 60:0.6073\n",
      "train auc: 0.943426521978\n",
      "test auc: 0.948823801162\n",
      "loss at iter 61:0.6037\n",
      "train auc: 0.943799251852\n",
      "test auc: 0.949153737969\n",
      "loss at iter 62:0.6002\n",
      "train auc: 0.944161790008\n",
      "test auc: 0.949468900273\n",
      "loss at iter 63:0.5967\n",
      "train auc: 0.944513454821\n",
      "test auc: 0.94977801781\n",
      "loss at iter 64:0.5934\n",
      "train auc: 0.944854879889\n",
      "test auc: 0.950074535701\n",
      "loss at iter 65:0.5901\n",
      "train auc: 0.945186722206\n",
      "test auc: 0.950365903986\n",
      "loss at iter 66:0.5870\n",
      "train auc: 0.945509031195\n",
      "test auc: 0.950647811542\n",
      "loss at iter 67:0.5839\n",
      "train auc: 0.945822921414\n",
      "test auc: 0.950922335242\n",
      "loss at iter 68:0.5809\n",
      "train auc: 0.946127283534\n",
      "test auc: 0.951190016715\n",
      "loss at iter 69:0.5780\n",
      "train auc: 0.946424287748\n",
      "test auc: 0.951451296704\n",
      "loss at iter 70:0.5751\n",
      "train auc: 0.94671318619\n",
      "test auc: 0.951702997415\n",
      "loss at iter 71:0.5723\n",
      "train auc: 0.94699424694\n",
      "test auc: 0.951944726015\n",
      "loss at iter 72:0.5696\n",
      "train auc: 0.947267951346\n",
      "test auc: 0.952183753076\n",
      "loss at iter 73:0.5669\n",
      "train auc: 0.947534555877\n",
      "test auc: 0.952416973103\n",
      "loss at iter 74:0.5643\n",
      "train auc: 0.947794608767\n",
      "test auc: 0.952642775903\n",
      "loss at iter 75:0.5617\n",
      "train auc: 0.948047738251\n",
      "test auc: 0.952865080535\n",
      "loss at iter 76:0.5592\n",
      "train auc: 0.948295433398\n",
      "test auc: 0.953078391554\n",
      "loss at iter 77:0.5568\n",
      "train auc: 0.948536786466\n",
      "test auc: 0.953287799124\n",
      "loss at iter 78:0.5544\n",
      "train auc: 0.948772591623\n",
      "test auc: 0.953494039845\n",
      "loss at iter 79:0.5521\n",
      "train auc: 0.949003234159\n",
      "test auc: 0.953691462585\n",
      "loss at iter 80:0.5498\n",
      "train auc: 0.949228751242\n",
      "test auc: 0.953884571432\n",
      "loss at iter 81:0.5475\n",
      "train auc: 0.949448252132\n",
      "test auc: 0.954071327425\n",
      "loss at iter 82:0.5453\n",
      "train auc: 0.949662895201\n",
      "test auc: 0.954255342897\n",
      "loss at iter 83:0.5432\n",
      "train auc: 0.949873348616\n",
      "test auc: 0.954437508186\n",
      "loss at iter 84:0.5410\n",
      "train auc: 0.950078997906\n",
      "test auc: 0.954613060588\n",
      "loss at iter 85:0.5390\n",
      "train auc: 0.950279871886\n",
      "test auc: 0.954787593374\n",
      "loss at iter 86:0.5369\n",
      "train auc: 0.950476723819\n",
      "test auc: 0.954957308696\n",
      "loss at iter 87:0.5349\n",
      "train auc: 0.95066960532\n",
      "test auc: 0.955122150327\n",
      "loss at iter 88:0.5330\n",
      "train auc: 0.950857805569\n",
      "test auc: 0.955285945103\n",
      "loss at iter 89:0.5311\n",
      "train auc: 0.951042694864\n",
      "test auc: 0.955445178056\n",
      "loss at iter 90:0.5292\n",
      "train auc: 0.95122313731\n",
      "test auc: 0.955601250484\n",
      "loss at iter 91:0.5273\n",
      "train auc: 0.951400242092\n",
      "test auc: 0.955753499124\n",
      "loss at iter 92:0.5255\n",
      "train auc: 0.95157368988\n",
      "test auc: 0.955901103625\n",
      "loss at iter 93:0.5237\n",
      "train auc: 0.951743899\n",
      "test auc: 0.956046151182\n",
      "loss at iter 94:0.5220\n",
      "train auc: 0.951910941838\n",
      "test auc: 0.956188038628\n",
      "loss at iter 95:0.5203\n",
      "train auc: 0.952074121583\n",
      "test auc: 0.956324841406\n",
      "loss at iter 96:0.5186\n",
      "train auc: 0.952234709207\n",
      "test auc: 0.956458466378\n",
      "loss at iter 97:0.5169\n",
      "train auc: 0.952391841008\n",
      "test auc: 0.956595170386\n",
      "loss at iter 98:0.5153\n",
      "train auc: 0.952546145578\n",
      "test auc: 0.956728743223\n",
      "loss at iter 99:0.5136\n",
      "train auc: 0.952697683426\n",
      "test auc: 0.956858878911\n",
      "loss at iter 100:0.5121\n",
      "train auc: 0.952846105245\n",
      "test auc: 0.956985660299\n",
      "loss at iter 101:0.5105\n",
      "train auc: 0.952991662622\n",
      "test auc: 0.957110739357\n",
      "loss at iter 102:0.5090\n",
      "train auc: 0.953134660326\n",
      "test auc: 0.95723293857\n",
      "loss at iter 103:0.5075\n",
      "train auc: 0.953275543867\n",
      "test auc: 0.957354086126\n",
      "loss at iter 104:0.5060\n",
      "train auc: 0.953413815316\n",
      "test auc: 0.957471653237\n",
      "loss at iter 105:0.5045\n",
      "train auc: 0.953550050303\n",
      "test auc: 0.95759024556\n",
      "loss at iter 106:0.5031\n",
      "train auc: 0.953683761963\n",
      "test auc: 0.95770653134\n",
      "loss at iter 107:0.5016\n",
      "train auc: 0.953814781099\n",
      "test auc: 0.957818387979\n",
      "loss at iter 108:0.5003\n",
      "train auc: 0.953944139484\n",
      "test auc: 0.957926649756\n",
      "loss at iter 109:0.4989\n",
      "train auc: 0.954071141674\n",
      "test auc: 0.958035778416\n",
      "loss at iter 110:0.4975\n",
      "train auc: 0.954196387743\n",
      "test auc: 0.958139332726\n",
      "loss at iter 111:0.4962\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train auc: 0.954319014263\n",
      "test auc: 0.958241260014\n",
      "loss at iter 112:0.4949\n",
      "train auc: 0.954439746685\n",
      "test auc: 0.958343557029\n",
      "loss at iter 113:0.4936\n",
      "train auc: 0.954558990952\n",
      "test auc: 0.958444920425\n",
      "loss at iter 114:0.4923\n",
      "train auc: 0.954676081508\n",
      "test auc: 0.958541997558\n",
      "loss at iter 115:0.4910\n",
      "train auc: 0.954791235086\n",
      "test auc: 0.958638568268\n",
      "loss at iter 116:0.4898\n",
      "train auc: 0.954903793997\n",
      "test auc: 0.958733694082\n",
      "loss at iter 117:0.4886\n",
      "train auc: 0.955015499253\n",
      "test auc: 0.958827871867\n",
      "loss at iter 118:0.4874\n",
      "train auc: 0.955125242428\n",
      "test auc: 0.958919587727\n",
      "loss at iter 119:0.4862\n",
      "train auc: 0.955232816304\n",
      "test auc: 0.959008126798\n",
      "loss at iter 120:0.4850\n",
      "train auc: 0.95533946515\n",
      "test auc: 0.959096920563\n",
      "loss at iter 121:0.4838\n",
      "train auc: 0.955444291502\n",
      "test auc: 0.959184549356\n",
      "loss at iter 122:0.4827\n",
      "train auc: 0.955547279908\n",
      "test auc: 0.959273028702\n",
      "loss at iter 123:0.4815\n",
      "train auc: 0.955648959325\n",
      "test auc: 0.959359552233\n",
      "loss at iter 124:0.4804\n",
      "train auc: 0.955749102985\n",
      "test auc: 0.959442415821\n",
      "loss at iter 125:0.4793\n",
      "train auc: 0.955847734875\n",
      "test auc: 0.959525357587\n",
      "loss at iter 126:0.4782\n",
      "train auc: 0.955944252255\n",
      "test auc: 0.959606147115\n",
      "loss at iter 127:0.4772\n",
      "train auc: 0.956040429931\n",
      "test auc: 0.959687279867\n",
      "loss at iter 128:0.4761\n",
      "train auc: 0.956135134452\n",
      "test auc: 0.959764912453\n",
      "loss at iter 129:0.4751\n",
      "train auc: 0.956228038741\n",
      "test auc: 0.95984227965\n",
      "loss at iter 130:0.4740\n",
      "train auc: 0.956319444125\n",
      "test auc: 0.959920288907\n",
      "loss at iter 131:0.4730\n",
      "train auc: 0.956410143002\n",
      "test auc: 0.959995930541\n",
      "loss at iter 132:0.4720\n",
      "train auc: 0.956499440755\n",
      "test auc: 0.960070360693\n",
      "loss at iter 133:0.4710\n",
      "train auc: 0.956587574593\n",
      "test auc: 0.960143840375\n",
      "loss at iter 134:0.4700\n",
      "train auc: 0.956674483362\n",
      "test auc: 0.960216963388\n",
      "loss at iter 135:0.4690\n",
      "train auc: 0.956760247355\n",
      "test auc: 0.96028945506\n",
      "loss at iter 136:0.4681\n",
      "train auc: 0.956845140511\n",
      "test auc: 0.960359164829\n",
      "loss at iter 137:0.4671\n",
      "train auc: 0.95692816068\n",
      "test auc: 0.960429044601\n",
      "loss at iter 138:0.4662\n",
      "train auc: 0.957010428096\n",
      "test auc: 0.960496185732\n",
      "loss at iter 139:0.4653\n",
      "train auc: 0.957091470995\n",
      "test auc: 0.960563330627\n",
      "loss at iter 140:0.4644\n",
      "train auc: 0.957171184118\n",
      "test auc: 0.960629105194\n",
      "loss at iter 141:0.4635\n",
      "train auc: 0.95725044893\n",
      "test auc: 0.960695653241\n",
      "loss at iter 142:0.4626\n",
      "train auc: 0.957328259869\n",
      "test auc: 0.960760313694\n",
      "loss at iter 143:0.4617\n",
      "train auc: 0.957405455901\n",
      "test auc: 0.960823968427\n",
      "loss at iter 144:0.4608\n",
      "train auc: 0.957481390261\n",
      "test auc: 0.960887088426\n",
      "loss at iter 145:0.4599\n",
      "train auc: 0.957556499899\n",
      "test auc: 0.960950931651\n",
      "loss at iter 146:0.4591\n",
      "train auc: 0.957630744777\n",
      "test auc: 0.961013325478\n",
      "loss at iter 147:0.4582\n",
      "train auc: 0.957703650993\n",
      "test auc: 0.961075625657\n",
      "loss at iter 148:0.4574\n",
      "train auc: 0.95777637045\n",
      "test auc: 0.961136935923\n",
      "loss at iter 149:0.4566\n",
      "train auc: 0.957847560565\n",
      "test auc: 0.96119610138\n",
      "loss at iter 150:0.4557\n",
      "train auc: 0.957918044127\n",
      "test auc: 0.961253722783\n",
      "loss at iter 151:0.4549\n",
      "train auc: 0.957987613868\n",
      "test auc: 0.961311718744\n",
      "loss at iter 152:0.4541\n",
      "train auc: 0.958056672887\n",
      "test auc: 0.961370146637\n",
      "loss at iter 153:0.4533\n",
      "train auc: 0.958124905154\n",
      "test auc: 0.961425887414\n",
      "loss at iter 154:0.4525\n",
      "train auc: 0.958192641086\n",
      "test auc: 0.961479912239\n",
      "loss at iter 155:0.4518\n",
      "train auc: 0.958259429251\n",
      "test auc: 0.961534666358\n",
      "loss at iter 156:0.4510\n",
      "train auc: 0.958325461028\n",
      "test auc: 0.961589752424\n",
      "loss at iter 157:0.4502\n",
      "train auc: 0.958390496112\n",
      "test auc: 0.961643453962\n",
      "loss at iter 158:0.4495\n",
      "train auc: 0.958455057032\n",
      "test auc: 0.96169515061\n",
      "loss at iter 159:0.4487\n",
      "train auc: 0.958518535818\n",
      "test auc: 0.961748582538\n",
      "loss at iter 160:0.4480\n",
      "train auc: 0.958581419445\n",
      "test auc: 0.961799036036\n",
      "loss at iter 161:0.4473\n",
      "train auc: 0.95864335203\n",
      "test auc: 0.96185173187\n",
      "loss at iter 162:0.4465\n",
      "train auc: 0.958704713295\n",
      "test auc: 0.961903442575\n",
      "loss at iter 163:0.4458\n",
      "train auc: 0.958765596231\n",
      "test auc: 0.961954284723\n",
      "loss at iter 164:0.4451\n",
      "train auc: 0.958825949651\n",
      "test auc: 0.962002454642\n",
      "loss at iter 165:0.4444\n",
      "train auc: 0.958885536913\n",
      "test auc: 0.962050917639\n",
      "loss at iter 166:0.4437\n",
      "train auc: 0.958944402558\n",
      "test auc: 0.962098927511\n",
      "loss at iter 167:0.4430\n",
      "train auc: 0.959002402664\n",
      "test auc: 0.962147903004\n",
      "loss at iter 168:0.4423\n",
      "train auc: 0.959059768959\n",
      "test auc: 0.962196092258\n",
      "loss at iter 169:0.4416\n",
      "train auc: 0.959116774926\n",
      "test auc: 0.962243097449\n",
      "loss at iter 170:0.4410\n",
      "train auc: 0.959173236786\n",
      "test auc: 0.962288567507\n",
      "loss at iter 171:0.4403\n",
      "train auc: 0.959228899672\n",
      "test auc: 0.962335278235\n",
      "loss at iter 172:0.4397\n",
      "train auc: 0.959284030593\n",
      "test auc: 0.962380799167\n",
      "loss at iter 173:0.4390\n",
      "train auc: 0.959338577433\n",
      "test auc: 0.962426382161\n",
      "loss at iter 174:0.4383\n",
      "train auc: 0.959392442684\n",
      "test auc: 0.962470665424\n",
      "loss at iter 175:0.4377\n",
      "train auc: 0.959445860215\n",
      "test auc: 0.96251430376\n",
      "loss at iter 176:0.4371\n",
      "train auc: 0.959498621045\n",
      "test auc: 0.962557854966\n",
      "loss at iter 177:0.4364\n",
      "train auc: 0.959550963705\n",
      "test auc: 0.962601610779\n",
      "loss at iter 178:0.4358\n",
      "train auc: 0.959602852391\n",
      "test auc: 0.962644851876\n",
      "loss at iter 179:0.4352\n",
      "train auc: 0.959654208615\n",
      "test auc: 0.962688317511\n",
      "loss at iter 180:0.4346\n",
      "train auc: 0.959705210261\n",
      "test auc: 0.962729972845\n",
      "loss at iter 181:0.4340\n",
      "train auc: 0.959755515578\n",
      "test auc: 0.962772768106\n",
      "loss at iter 182:0.4334\n",
      "train auc: 0.959805374342\n",
      "test auc: 0.962812855705\n",
      "loss at iter 183:0.4328\n",
      "train auc: 0.959854584312\n",
      "test auc: 0.962852489872\n",
      "loss at iter 184:0.4322\n",
      "train auc: 0.959903439789\n",
      "test auc: 0.96289355727\n",
      "loss at iter 185:0.4316\n",
      "train auc: 0.959951943785\n",
      "test auc: 0.962932132249\n",
      "loss at iter 186:0.4310\n",
      "train auc: 0.960000302945\n",
      "test auc: 0.962970173804\n",
      "loss at iter 187:0.4304\n",
      "train auc: 0.960047609871\n",
      "test auc: 0.963008790973\n",
      "loss at iter 188:0.4299\n",
      "train auc: 0.960094295877\n",
      "test auc: 0.963048006086\n",
      "loss at iter 189:0.4293\n",
      "train auc: 0.960141213908\n",
      "test auc: 0.963083972375\n",
      "loss at iter 190:0.4287\n",
      "train auc: 0.960187230322\n",
      "test auc: 0.96312224754\n",
      "loss at iter 191:0.4282\n",
      "train auc: 0.960233235594\n",
      "test auc: 0.963160617933\n",
      "loss at iter 192:0.4276\n",
      "train auc: 0.960278825342\n",
      "test auc: 0.963198951098\n",
      "loss at iter 193:0.4271\n",
      "train auc: 0.960323736294\n",
      "test auc: 0.963236148816\n",
      "loss at iter 194:0.4265\n",
      "train auc: 0.960368289844\n",
      "test auc: 0.963273271272\n",
      "loss at iter 195:0.4260\n",
      "train auc: 0.960412390854\n",
      "test auc: 0.963310121942\n",
      "loss at iter 196:0.4255\n",
      "train auc: 0.960456286663\n",
      "test auc: 0.963345736679\n",
      "loss at iter 197:0.4249\n",
      "train auc: 0.960499762549\n",
      "test auc: 0.96338275029\n",
      "loss at iter 198:0.4244\n",
      "train auc: 0.96054246462\n",
      "test auc: 0.963418672246\n",
      "loss at iter 199:0.4239\n",
      "train auc: 0.96058527777\n",
      "test auc: 0.963455615401\n",
      "loss at iter 200:0.4234\n",
      "train auc: 0.960627540424\n",
      "test auc: 0.963490830626\n",
      "loss at iter 201:0.4229\n",
      "train auc: 0.960669219956\n",
      "test auc: 0.963524852882\n",
      "loss at iter 202:0.4223\n",
      "train auc: 0.960710692236\n",
      "test auc: 0.96355689709\n",
      "loss at iter 203:0.4218\n",
      "train auc: 0.960752038047\n",
      "test auc: 0.963589410709\n",
      "loss at iter 204:0.4213\n",
      "train auc: 0.960793122566\n",
      "test auc: 0.963622688269\n",
      "loss at iter 205:0.4208\n",
      "train auc: 0.960833353205\n",
      "test auc: 0.963656635714\n",
      "loss at iter 206:0.4203\n",
      "train auc: 0.960873232362\n",
      "test auc: 0.963689626474\n",
      "loss at iter 207:0.4198\n",
      "train auc: 0.96091278501\n",
      "test auc: 0.963722374247\n",
      "loss at iter 208:0.4194\n",
      "train auc: 0.960951857571\n",
      "test auc: 0.963754259923\n",
      "loss at iter 209:0.4189\n",
      "train auc: 0.960990704749\n",
      "test auc: 0.963785605178\n",
      "loss at iter 210:0.4184\n",
      "train auc: 0.961029535225\n",
      "test auc: 0.96381707973\n",
      "loss at iter 211:0.4179\n",
      "train auc: 0.961067926771\n",
      "test auc: 0.963847386356\n",
      "loss at iter 212:0.4174\n",
      "train auc: 0.961106189837\n",
      "test auc: 0.963877424097\n",
      "loss at iter 213:0.4170\n",
      "train auc: 0.961144046829\n",
      "test auc: 0.963909742732\n",
      "loss at iter 214:0.4165\n",
      "train auc: 0.961181081467\n",
      "test auc: 0.963939887834\n",
      "loss at iter 215:0.4161\n",
      "train auc: 0.961218292419\n",
      "test auc: 0.963969970307\n",
      "loss at iter 216:0.4156\n",
      "train auc: 0.961255289232\n",
      "test auc: 0.964000813203\n",
      "loss at iter 217:0.4151\n",
      "train auc: 0.96129186497\n",
      "test auc: 0.964030781757\n",
      "loss at iter 218:0.4147\n",
      "train auc: 0.961327940547\n",
      "test auc: 0.964060421449\n",
      "loss at iter 219:0.4142\n",
      "train auc: 0.961363982791\n",
      "test auc: 0.964090064929\n",
      "loss at iter 220:0.4138\n",
      "train auc: 0.961399604287\n",
      "test auc: 0.964120464009\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss at iter 221:0.4133\n",
      "train auc: 0.961434899067\n",
      "test auc: 0.964150471222\n",
      "loss at iter 222:0.4129\n",
      "train auc: 0.961469546206\n",
      "test auc: 0.964176345075\n",
      "loss at iter 223:0.4125\n",
      "train auc: 0.96150471993\n",
      "test auc: 0.964204426963\n",
      "loss at iter 224:0.4120\n",
      "train auc: 0.961539204738\n",
      "test auc: 0.964232472418\n",
      "loss at iter 225:0.4116\n",
      "train auc: 0.961573829663\n",
      "test auc: 0.96425977592\n",
      "loss at iter 226:0.4112\n",
      "train auc: 0.961608261118\n",
      "test auc: 0.964287746386\n",
      "loss at iter 227:0.4108\n",
      "train auc: 0.961641906614\n",
      "test auc: 0.964315935495\n",
      "loss at iter 228:0.4103\n",
      "train auc: 0.961675589071\n",
      "test auc: 0.964344139244\n",
      "loss at iter 229:0.4099\n",
      "train auc: 0.961708916843\n",
      "test auc: 0.964370944973\n",
      "loss at iter 230:0.4095\n",
      "train auc: 0.961742002054\n",
      "test auc: 0.964397489059\n",
      "loss at iter 231:0.4091\n",
      "train auc: 0.961774592292\n",
      "test auc: 0.964422850677\n",
      "loss at iter 232:0.4087\n",
      "train auc: 0.961806981463\n",
      "test auc: 0.964447594916\n",
      "loss at iter 233:0.4083\n",
      "train auc: 0.961839145128\n",
      "test auc: 0.964473052272\n",
      "loss at iter 234:0.4079\n",
      "train auc: 0.961870938779\n",
      "test auc: 0.964498152098\n",
      "loss at iter 235:0.4075\n",
      "train auc: 0.961902505751\n",
      "test auc: 0.964524779102\n",
      "loss at iter 236:0.4071\n",
      "train auc: 0.961933696321\n",
      "test auc: 0.964550374385\n",
      "loss at iter 237:0.4067\n",
      "train auc: 0.961965275691\n",
      "test auc: 0.964576401227\n",
      "loss at iter 238:0.4063\n",
      "train auc: 0.961996371346\n",
      "test auc: 0.964601215994\n",
      "loss at iter 239:0.4059\n",
      "train auc: 0.962027155794\n",
      "test auc: 0.964626414447\n",
      "loss at iter 240:0.4055\n",
      "train auc: 0.962057864582\n",
      "test auc: 0.96465039169\n",
      "loss at iter 241:0.4051\n",
      "train auc: 0.962088357694\n",
      "test auc: 0.9646753861\n",
      "loss at iter 242:0.4047\n",
      "train auc: 0.962118783985\n",
      "test auc: 0.96469960388\n",
      "loss at iter 243:0.4043\n",
      "train auc: 0.962149147467\n",
      "test auc: 0.964724360079\n",
      "loss at iter 244:0.4040\n",
      "train auc: 0.962178965592\n",
      "test auc: 0.964748372462\n",
      "loss at iter 245:0.4036\n",
      "train auc: 0.962208674349\n",
      "test auc: 0.964773115747\n",
      "loss at iter 246:0.4032\n",
      "train auc: 0.962237737625\n",
      "test auc: 0.964795622251\n",
      "loss at iter 247:0.4028\n",
      "train auc: 0.962267056798\n",
      "test auc: 0.964819034152\n",
      "loss at iter 248:0.4025\n",
      "train auc: 0.96229589055\n",
      "test auc: 0.964840455418\n",
      "loss at iter 249:0.4021\n",
      "train auc: 0.962324529188\n",
      "test auc: 0.964863859366\n",
      "loss at iter 250:0.4017\n",
      "train auc: 0.962353093646\n",
      "test auc: 0.964887955157\n",
      "loss at iter 251:0.4014\n",
      "train auc: 0.962381483721\n",
      "test auc: 0.964910932752\n",
      "loss at iter 252:0.4010\n",
      "train auc: 0.962409497401\n",
      "test auc: 0.964932755847\n",
      "loss at iter 253:0.4006\n",
      "train auc: 0.962437777082\n",
      "test auc: 0.964955316605\n",
      "loss at iter 254:0.4003\n",
      "train auc: 0.962465296776\n",
      "test auc: 0.964977800702\n",
      "loss at iter 255:0.3999\n",
      "train auc: 0.962493186132\n",
      "test auc: 0.965000708851\n",
      "loss at iter 256:0.3996\n",
      "train auc: 0.962520647461\n",
      "test auc: 0.965023245546\n",
      "loss at iter 257:0.3992\n",
      "train auc: 0.962547977815\n",
      "test auc: 0.965045140863\n",
      "loss at iter 258:0.3989\n",
      "train auc: 0.9625751226\n",
      "test auc: 0.965066415977\n",
      "loss at iter 259:0.3985\n",
      "train auc: 0.962602052834\n",
      "test auc: 0.96508877689\n",
      "loss at iter 260:0.3982\n",
      "train auc: 0.962628840962\n",
      "test auc: 0.965112404511\n",
      "loss at iter 261:0.3979\n",
      "train auc: 0.962655224627\n",
      "test auc: 0.965133555263\n",
      "loss at iter 262:0.3975\n",
      "train auc: 0.962681569835\n",
      "test auc: 0.965154746878\n",
      "loss at iter 263:0.3972\n",
      "train auc: 0.962707607794\n",
      "test auc: 0.965175269486\n",
      "loss at iter 264:0.3968\n",
      "train auc: 0.962733980428\n",
      "test auc: 0.965196198865\n",
      "loss at iter 265:0.3965\n",
      "train auc: 0.962759759785\n",
      "test auc: 0.965217149989\n",
      "loss at iter 266:0.3962\n",
      "train auc: 0.962785569108\n",
      "test auc: 0.965238462\n",
      "loss at iter 267:0.3959\n",
      "train auc: 0.962811025486\n",
      "test auc: 0.965258674835\n",
      "loss at iter 268:0.3955\n",
      "train auc: 0.962836145176\n",
      "test auc: 0.965278693476\n",
      "loss at iter 269:0.3952\n",
      "train auc: 0.962861264796\n",
      "test auc: 0.96529992102\n",
      "loss at iter 270:0.3949\n",
      "train auc: 0.962886329168\n",
      "test auc: 0.965321286169\n",
      "loss at iter 271:0.3946\n",
      "train auc: 0.962911240183\n",
      "test auc: 0.965340857906\n",
      "loss at iter 272:0.3942\n",
      "train auc: 0.962935883927\n",
      "test auc: 0.965360485948\n",
      "loss at iter 273:0.3939\n",
      "train auc: 0.962960279115\n",
      "test auc: 0.965381624455\n",
      "loss at iter 274:0.3936\n",
      "train auc: 0.962984640972\n",
      "test auc: 0.965400494728\n",
      "loss at iter 275:0.3933\n",
      "train auc: 0.963008780775\n",
      "test auc: 0.965419200608\n",
      "loss at iter 276:0.3930\n",
      "train auc: 0.963033035197\n",
      "test auc: 0.965438550385\n",
      "loss at iter 277:0.3927\n",
      "train auc: 0.963056975079\n",
      "test auc: 0.965457362309\n",
      "loss at iter 278:0.3923\n",
      "train auc: 0.963080713938\n",
      "test auc: 0.96547636761\n",
      "loss at iter 279:0.3920\n",
      "train auc: 0.963104493175\n",
      "test auc: 0.965494376527\n",
      "loss at iter 280:0.3917\n",
      "train auc: 0.963127901114\n",
      "test auc: 0.965513737214\n",
      "loss at iter 281:0.3914\n",
      "train auc: 0.963151255815\n",
      "test auc: 0.965533261273\n",
      "loss at iter 282:0.3911\n",
      "train auc: 0.963174528162\n",
      "test auc: 0.965552582391\n",
      "loss at iter 283:0.3908\n",
      "train auc: 0.963197721592\n",
      "test auc: 0.965571600762\n",
      "loss at iter 284:0.3905\n",
      "train auc: 0.963220683621\n",
      "test auc: 0.965589839177\n",
      "loss at iter 285:0.3902\n",
      "train auc: 0.963243542362\n",
      "test auc: 0.965608891872\n",
      "loss at iter 286:0.3899\n",
      "train auc: 0.963266297618\n",
      "test auc: 0.965626517516\n",
      "loss at iter 287:0.3896\n",
      "train auc: 0.963288907822\n",
      "test auc: 0.965643714321\n",
      "loss at iter 288:0.3893\n",
      "train auc: 0.963311663259\n",
      "test auc: 0.965661379551\n",
      "loss at iter 289:0.3891\n",
      "train auc: 0.963333975024\n",
      "test auc: 0.965680378787\n",
      "loss at iter 290:0.3888\n",
      "train auc: 0.963356275893\n",
      "test auc: 0.965697384901\n",
      "loss at iter 291:0.3885\n",
      "train auc: 0.963378303142\n",
      "test auc: 0.965715041753\n",
      "loss at iter 292:0.3882\n",
      "train auc: 0.963400619861\n",
      "test auc: 0.96573204577\n",
      "loss at iter 293:0.3879\n",
      "train auc: 0.963422388091\n",
      "test auc: 0.96574938802\n",
      "loss at iter 294:0.3876\n",
      "train auc: 0.96344433026\n",
      "test auc: 0.965766153511\n",
      "loss at iter 295:0.3873\n",
      "train auc: 0.963465991164\n",
      "test auc: 0.965785079616\n",
      "loss at iter 296:0.3871\n",
      "train auc: 0.963487674203\n",
      "test auc: 0.965802165333\n",
      "loss at iter 297:0.3868\n",
      "train auc: 0.963508953164\n",
      "test auc: 0.965817942254\n",
      "loss at iter 298:0.3865\n",
      "train auc: 0.963530228825\n",
      "test auc: 0.965835993115\n",
      "loss at iter 299:0.3862\n",
      "train auc: 0.963551205152\n",
      "test auc: 0.965852785426\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "s.run(tf.global_variables_initializer())\n",
    "\n",
    "for i in range(300):\n",
    "    #batchX, batchY = s.run(tf.train.batch([X_train_flat, y_train_oh],100,enqueue_many=True, capacity=1))\n",
    "    s.run(optimizer, {input_X: X_train_flat, input_y: y_train_oh})\n",
    "    #s.run(optimizer, {input_X: batchX, input_y: batchY})\n",
    "    loss_i = s.run(loss, {input_X: X_train_flat, input_y: y_train_oh})\n",
    "    print(\"loss at iter %i:%.4f\" % (i, loss_i))\n",
    "    print(\"train auc:\", roc_auc_score(y_train_oh, s.run(predicted_y, {input_X:X_train_flat})))\n",
    "    print(\"test auc:\", roc_auc_score(y_test_oh, s.run(predicted_y, {input_X:X_test_flat})))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8868\n",
      "0.8993\n",
      "0.8943\n"
     ]
    }
   ],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(predicted_y,1), tf.argmax(input_y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(s.run(accuracy, feed_dict={input_X:X_train_flat, input_y: y_train_oh}))\n",
    "print(s.run(accuracy, feed_dict={input_X:X_val_flat, input_y: y_val_oh}))\n",
    "print(s.run(accuracy, feed_dict={input_X:X_test_flat, input_y: y_test_oh}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}