build_mnist_dcgan_and_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "intimate-convert",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow==2.4.1 -U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "large-uganda",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from tensorflow.keras.models import load_model\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pleased-philippines",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "partial-affect",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_X_PATH = './train_X.npy'\n",
    "TEST_X_PATH = './test_X.npy'\n",
    "TRAIN_Y_PATH = './train_y.npy'\n",
    "TEST_Y_PATH = './test_y.npy'\n",
    "(train_X, train_y), (test_X, test_y) = mnist.load_data()\n",
    "train_X = (train_X-127.5)/127.5\n",
    "test_X = (test_X-127.5)/127.5\n",
    "# channel last\n",
    "train_X = train_X.reshape((train_X.shape[0],train_X.shape[1],train_X.shape[2],1))\n",
    "test_X = test_X.reshape((test_X.shape[0],test_X.shape[1],test_X.shape[2],1))\n",
    "# one-hot\n",
    "train_y = np.eye(10)[train_y]\n",
    "test_y = np.eye(10)[test_y]\n",
    "np.save(TRAIN_X_PATH,train_X)\n",
    "np.save(TEST_X_PATH,test_X)\n",
    "np.save(TRAIN_Y_PATH,train_y)\n",
    "np.save(TEST_Y_PATH,test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "valuable-replica",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X.shape,train_y.shape,test_X.shape,test_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "limiting-nebraska",
   "metadata": {},
   "outputs": [],
   "source": [
    "role = sagemaker.get_execution_role()\n",
    "sess = sagemaker.session.Session()\n",
    "train_X_uri = sess.upload_data(path=TRAIN_X_PATH, bucket=sess.default_bucket(), key_prefix='sagemaker/mnist')\n",
    "train_y_uri = sess.upload_data(path=TRAIN_Y_PATH, bucket=sess.default_bucket(), key_prefix='sagemaker/mnist')\n",
    "test_X_uri = sess.upload_data(path=TEST_X_PATH, bucket=sess.default_bucket(), key_prefix='sagemaker/mnist')\n",
    "test_y_uri = sess.upload_data(path=TEST_Y_PATH, bucket=sess.default_bucket(), key_prefix='sagemaker/mnist')\n",
    "print(train_X_uri)\n",
    "print(train_y_uri)\n",
    "print(test_X_uri)\n",
    "print(test_y_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "working-seating",
   "metadata": {},
   "source": [
    "## DCGAN Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "extraordinary-pantyhose",
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = TensorFlow(\n",
    "    entry_point='./src/dcgan_train.py',\n",
    "    role=role,\n",
    "    instance_count=1,\n",
    "    instance_type='ml.p3.2xlarge',\n",
    "    framework_version='2.4.1',\n",
    "    py_version='py37',\n",
    "    hyperparameters={\n",
    "        'sagemaker_s3_output':'s3://sagemaker-ap-northeast-1-155580384669/mnist_dcgan_intermediate',\n",
    "        'epochs':5,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "potential-worth",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "estimator.fit({\n",
    "    'train': train_X_uri,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "metric-hamburg",
   "metadata": {},
   "outputs": [],
   "source": [
    "generator_model_uri = estimator.latest_training_job.describe()['ModelArtifacts']['S3ModelArtifacts']\n",
    "print(generator_model_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wanted-riverside",
   "metadata": {},
   "source": [
    "## Classifier Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ranking-track",
   "metadata": {},
   "outputs": [],
   "source": [
    "estimator = TensorFlow(\n",
    "    entry_point='./src/classifier_train.py',\n",
    "    role=role,\n",
    "    instance_count=1,\n",
    "    instance_type='ml.p3.2xlarge',\n",
    "    framework_version='2.4.1',\n",
    "    py_version='py37',\n",
    "    hyperparameters={\n",
    "        'epochs':10,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "derived-start",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "print(train_X_uri[:-11])\n",
    "\n",
    "estimator.fit({\n",
    "    'train': train_X_uri[:-11],\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "written-flooring",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_model_uri = estimator.latest_training_job.describe()['ModelArtifacts']['S3ModelArtifacts']\n",
    "print(classifier_model_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "selective-mattress",
   "metadata": {},
   "source": [
    "## Check the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "naughty-cuisine",
   "metadata": {},
   "outputs": [],
   "source": [
    "!aws s3 cp {generator_model_uri} .\n",
    "!mkdir -p ./src/ggv2/components/artifacts/com.example.Publisher/1.0.0\n",
    "!tar zxvf model.tar.gz -C ./src/ggv2/components/artifacts/com.example.Publisher/1.0.0\n",
    "!rm model.tar.gz \n",
    "!aws s3 cp {classifier_model_uri} .\n",
    "!mkdir -p ./src/ggv2/components/artifacts/com.example.Subscriber/1.0.0\n",
    "!mkdir -p ./src/ggv2/components/artifacts/com.example.Subscriber/1.0.1\n",
    "!tar zxvf model.tar.gz -C ./src/ggv2/components/artifacts/com.example.Subscriber/1.0.0\n",
    "!mv ./src/ggv2/components/artifacts/com.example.Subscriber/1.0.0/2.h5 ./src/ggv2/components/artifacts/com.example.Subscriber/1.0.1/2.h5\n",
    "!rm model.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dependent-commonwealth",
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = tf.keras.models.load_model('./src/ggv2/components/artifacts/com.example.Publisher/1.0.0/1.h5')\n",
    "classifier1 = tf.keras.models.load_model('./src/ggv2/components/artifacts/com.example.Subscriber/1.0.0/1.h5')\n",
    "classifier2 = tf.keras.models.load_model('./src/ggv2/components/artifacts/com.example.Subscriber/1.0.1/2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sharing-southeast",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "generator.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "micro-activity",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "classifier.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "atomic-configuration",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_X = np.random.uniform(-1, 1, (1,7,7,1))\n",
    "pred_y = generator.predict(pred_X)\n",
    "from matplotlib import pyplot as plt\n",
    "plt.imshow(pred_y[0,:,:,0],'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "higher-relief",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.argmax(classifier.predict(pred_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sixth-grain",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_amazonei_tensorflow2_p36",
   "language": "python",
   "name": "conda_amazonei_tensorflow2_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}