{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras and Tensor flow Introduction\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.2.4-tf'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.1.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = './dataset/train/'\n",
    "test_dir = './dataset/test/'\n",
    "\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img\n",
    "train_datagen = ImageDataGenerator(rescale=1./255,\n",
    "                                   zoom_range=0.3,\n",
    "                                   horizontal_flip=True)\n",
    "\n",
    "training_set = train_datagen.flow_from_directory(train_dir,\n",
    "                                                batch_size=64,\n",
    "                                                target_size=(48,48),\n",
    "                                                shuffle=True,\n",
    "                                                color_mode='grayscale',\n",
    "                                                class_mode='categorical')\n",
    "\n",
    "validation_datagen = ImageDataGenerator(rescale=1./255)\n",
    "validation_set = validation_datagen.flow_from_directory(test_dir,\n",
    "                                                batch_size=64,\n",
    "                                                target_size=(48,48),\n",
    "                                                shuffle=True,\n",
    "                                                color_mode='grayscale',\n",
    "                                                class_mode='categorical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense,Dropout,BatchNormalization\n",
    "from tensorflow.keras import regularizers\n",
    "model = Sequential([\n",
    "    Conv2D(32, (3, 3), activation='relu', input_shape=(48, 48,1)),\n",
    "    MaxPooling2D(pool_size=(2, 2)),\n",
    "    Flatten(),\n",
    "    Dense(7, activation='softmax')\n",
    "])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 46, 46, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 23, 23, 32)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 16928)             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 7)                 118503    \n",
      "=================================================================\n",
      "Total params: 118,823\n",
      "Trainable params: 118,823\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.optimizers import Adam, RMSprop, SGD    \n",
    "model.compile(optimizer=Adam(lr=0.0001, decay=1e-6), \n",
    "                  loss='categorical_crossentropy', \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "#model.compile(optimizer=keras.optimizers.SGD(lr=0.001),\n",
    "#              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "#              metrics=['accuracy'])\n",
    "\n",
    "#model.compile(loss='sparse_categorical_crossentropy',optimizer='sgd',metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'.\\\\my_newlogs2\\\\run_2020_12_02_20_00_41'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create a function that creates a unique folder for training logs\n",
    "import os\n",
    "root_logdir = os.path.join(os.curdir,\"my_newlogs2\")\n",
    "def get_run_logdir():\n",
    "    import time\n",
    "    run_id=time.strftime(\"run_%Y_%m_%d_%H_%M_%S\")\n",
    "    return os.path.join(root_logdir,run_id)\n",
    "run_logdir=get_run_logdir()\n",
    "run_logdir\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "WARNING:tensorflow:sample_weight modes were coerced from\n",
      "  ...\n",
      "    to  \n",
      "  ['...']\n",
      "Train for 448 steps, validate for 112 steps\n",
      "Epoch 1/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.7827 - accuracy: 0.2682\n",
      "Epoch 00001: val_loss improved from inf to 1.73542, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 327s 731ms/step - loss: 1.7827 - accuracy: 0.2680 - val_loss: 1.7354 - val_accuracy: 0.3168\n",
      "Epoch 2/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.7183 - accuracy: 0.3109\n",
      "Epoch 00002: val_loss improved from 1.73542 to 1.67081, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 61ms/step - loss: 1.7184 - accuracy: 0.3111 - val_loss: 1.6708 - val_accuracy: 0.3657\n",
      "Epoch 3/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6930 - accuracy: 0.3338\n",
      "Epoch 00003: val_loss improved from 1.67081 to 1.64876, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 130s 291ms/step - loss: 1.6929 - accuracy: 0.3342 - val_loss: 1.6488 - val_accuracy: 0.3809\n",
      "Epoch 4/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6757 - accuracy: 0.3459\n",
      "Epoch 00004: val_loss improved from 1.64876 to 1.62627, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 60ms/step - loss: 1.6757 - accuracy: 0.3461 - val_loss: 1.6263 - val_accuracy: 0.3782\n",
      "Epoch 5/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.6616 - accuracy: 0.3500\n",
      "Epoch 00005: val_loss improved from 1.62627 to 1.61856, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.6617 - accuracy: 0.3501 - val_loss: 1.6186 - val_accuracy: 0.3923\n",
      "Epoch 6/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6519 - accuracy: 0.3568\n",
      "Epoch 00006: val_loss improved from 1.61856 to 1.60630, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.6516 - accuracy: 0.3569 - val_loss: 1.6063 - val_accuracy: 0.3804\n",
      "Epoch 7/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6445 - accuracy: 0.3595\n",
      "Epoch 00007: val_loss improved from 1.60630 to 1.59449, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.6449 - accuracy: 0.3594 - val_loss: 1.5945 - val_accuracy: 0.3998\n",
      "Epoch 8/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6377 - accuracy: 0.3630\n",
      "Epoch 00008: val_loss improved from 1.59449 to 1.59087, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.6377 - accuracy: 0.3630 - val_loss: 1.5909 - val_accuracy: 0.3910\n",
      "Epoch 9/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.6304 - accuracy: 0.3666\n",
      "Epoch 00009: val_loss improved from 1.59087 to 1.57934, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.6301 - accuracy: 0.3669 - val_loss: 1.5793 - val_accuracy: 0.4009\n",
      "Epoch 10/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6229 - accuracy: 0.3769\n",
      "Epoch 00010: val_loss improved from 1.57934 to 1.57710, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 60ms/step - loss: 1.6232 - accuracy: 0.3768 - val_loss: 1.5771 - val_accuracy: 0.3987\n",
      "Epoch 11/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6201 - accuracy: 0.3728\n",
      "Epoch 00011: val_loss did not improve from 1.57710\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.6201 - accuracy: 0.3728 - val_loss: 1.5843 - val_accuracy: 0.3972\n",
      "Epoch 12/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6136 - accuracy: 0.3778\n",
      "Epoch 00012: val_loss improved from 1.57710 to 1.56520, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.6132 - accuracy: 0.3779 - val_loss: 1.5652 - val_accuracy: 0.4016\n",
      "Epoch 13/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6107 - accuracy: 0.3782\n",
      "Epoch 00013: val_loss improved from 1.56520 to 1.56176, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.6107 - accuracy: 0.3781 - val_loss: 1.5618 - val_accuracy: 0.4067\n",
      "Epoch 14/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.6051 - accuracy: 0.3816\n",
      "Epoch 00014: val_loss improved from 1.56176 to 1.54801, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.6051 - accuracy: 0.3817 - val_loss: 1.5480 - val_accuracy: 0.4090\n",
      "Epoch 15/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.6016 - accuracy: 0.3833\n",
      "Epoch 00015: val_loss did not improve from 1.54801\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.6016 - accuracy: 0.3833 - val_loss: 1.5491 - val_accuracy: 0.4149\n",
      "Epoch 16/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.5975 - accuracy: 0.3871\n",
      "Epoch 00016: val_loss did not improve from 1.54801\n",
      "448/448 [==============================] - 27s 59ms/step - loss: 1.5975 - accuracy: 0.3868 - val_loss: 1.5511 - val_accuracy: 0.4114\n",
      "Epoch 17/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.5939 - accuracy: 0.3864\n",
      "Epoch 00017: val_loss improved from 1.54801 to 1.54395, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 28s 63ms/step - loss: 1.5935 - accuracy: 0.3867 - val_loss: 1.5439 - val_accuracy: 0.4064\n",
      "Epoch 18/60\n",
      "447/448 [============================>.] - ETA: 1s - loss: 1.5886 - accuracy: 0.3893\n",
      "Epoch 00018: val_loss improved from 1.54395 to 1.53865, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 1031s 2s/step - loss: 1.5886 - accuracy: 0.3893 - val_loss: 1.5387 - val_accuracy: 0.4189\n",
      "Epoch 19/60\n",
      "447/448 [============================>.] - ETA: 1s - loss: 1.5857 - accuracy: 0.3903\n",
      "Epoch 00019: val_loss improved from 1.53865 to 1.53270, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 677s 2s/step - loss: 1.5854 - accuracy: 0.3905 - val_loss: 1.5327 - val_accuracy: 0.4160\n",
      "Epoch 20/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.5829 - accuracy: 0.3913\n",
      "Epoch 00020: val_loss did not improve from 1.53270\n",
      "448/448 [==============================] - 36s 80ms/step - loss: 1.5831 - accuracy: 0.3912 - val_loss: 1.5436 - val_accuracy: 0.4097\n",
      "Epoch 21/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.5782 - accuracy: 0.3971\n",
      "Epoch 00021: val_loss did not improve from 1.53270\n",
      "448/448 [==============================] - 26s 58ms/step - loss: 1.5784 - accuracy: 0.3968 - val_loss: 1.5368 - val_accuracy: 0.4163\n",
      "Epoch 22/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.5781 - accuracy: 0.3957\n",
      "Epoch 00022: val_loss improved from 1.53270 to 1.51980, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 26s 59ms/step - loss: 1.5779 - accuracy: 0.3958 - val_loss: 1.5198 - val_accuracy: 0.4283\n",
      "Epoch 23/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.5694 - accuracy: 0.3984\n",
      "Epoch 00023: val_loss improved from 1.51980 to 1.51544, saving model to MLP_model.h5\n",
      "448/448 [==============================] - 26s 58ms/step - loss: 1.5695 - accuracy: 0.3983 - val_loss: 1.5154 - val_accuracy: 0.4247\n",
      "Epoch 24/60\n",
      "446/448 [============================>.] - ETA: 0s - loss: 1.5677 - accuracy: 0.4030\n",
      "Epoch 00024: val_loss did not improve from 1.51544\n",
      "448/448 [==============================] - 26s 58ms/step - loss: 1.5679 - accuracy: 0.4028 - val_loss: 1.5276 - val_accuracy: 0.4213\n",
      "Epoch 25/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.5663 - accuracy: 0.4016\n",
      "Epoch 00025: val_loss did not improve from 1.51544\n",
      "448/448 [==============================] - 26s 58ms/step - loss: 1.5660 - accuracy: 0.4018 - val_loss: 1.5197 - val_accuracy: 0.4210\n",
      "Epoch 26/60\n",
      "447/448 [============================>.] - ETA: 0s - loss: 1.5636 - accuracy: 0.4033\n",
      "Epoch 00026: val_loss did not improve from 1.51544\n",
      "Restoring model weights from the end of the best epoch.\n",
      "448/448 [==============================] - 26s 58ms/step - loss: 1.5637 - accuracy: 0.4034 - val_loss: 1.5243 - val_accuracy: 0.4242\n",
      "Epoch 00026: early stopping\n"
     ]
    }
   ],
   "source": [
    "#create a tensorboard callback function that makes use of folder logger\n",
    "tensorboard_cb=keras.callbacks.TensorBoard(run_logdir)\n",
    "\n",
    "#create checkpoint callback that saves every epoch\n",
    "checkpoint_cb = keras.callbacks.ModelCheckpoint(\"MLP_model.h5\",\n",
    "    save_best_only=True, verbose=1,mode='min', moniter='val_loss')\n",
    "\n",
    "#create early stopping callback\n",
    "\n",
    "early_stopping_cb = keras.callbacks.EarlyStopping(monitor='val_loss', \n",
    "                          min_delta=0, \n",
    "                          patience=3, \n",
    "                          verbose=1,\n",
    "                          restore_best_weights=True)\n",
    "\n",
    "steps_per_epoch = training_set.n // training_set.batch_size\n",
    "validation_steps = validation_set.n // validation_set.batch_size\n",
    "\n",
    "hist = model.fit(x=training_set,\n",
    "                 validation_data=validation_set,\n",
    "                 epochs=60,\n",
    "                 callbacks=[tensorboard_cb,\n",
    "            checkpoint_cb, early_stopping_cb],\n",
    "                 steps_per_epoch=steps_per_epoch,\n",
    "                 validation_steps=validation_steps)\n",
    "\n",
    "                                                \n",
    "                                                \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#end of training - evaluate now using unused test data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}