u-net-version_1.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## U-net model\n",
    "\n",
    "This notebook contains the code for initializing and training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def restart_kernel(restart=False):\n",
    "    import IPython\n",
    "    app = IPython.Application.instance()\n",
    "    if restart:\n",
    "        app.kernel.do_shutdown(True)\n",
    "\n",
    "restart_kernel(False)\n",
    "\n",
    "import os\n",
    "os.environ[\"THEANO_FLAGS\"] = \"device=cpu,lib.cnmem=0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import keras\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, Concatenate, Activation, Conv2D, Conv2DTranspose, MaxPooling2D, UpSampling2D, ZeroPadding2D, Cropping2D, BatchNormalization, Dropout\n",
    "from keras.optimizers import Adadelta, Adam\n",
    "from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau\n",
    "from keras import backend as K\n",
    "from keras.utils.io_utils import HDF5Matrix\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "#import skimage.exposure\n",
    "#import cv2\n",
    "import h5py\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from pushbullet import Pushbullet\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.tools as tls\n",
    "import plotly.graph_objs as go\n",
    "import time\n",
    "import threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Define the push function that can be called later\n",
    "def push(title='Done!',text=''):\n",
    "    Pushbullet('o.YFPNNPfGRekivaCGHa4qMSgjZt8zJ6FL').devices[0].push_note(title,text)\n",
    "def push_url(title='Link',url=''):\n",
    "    Pushbullet('o.YFPNNPfGRekivaCGHa4qMSgjZt8zJ6FL').devices[0].push_link(title,url)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is run # 39\n"
     ]
    }
   ],
   "source": [
    "# Increment the counter\n",
    "def counter():\n",
    "    run = np.load('./data/misc/run_counter.npy')\n",
    "    run += 1\n",
    "    np.save('./data/misc/run_counter.npy',run)\n",
    "    return run\n",
    "run = counter()\n",
    "\n",
    "# Set the counter to a specific value\n",
    "def set_counter(run):\n",
    "    run = run\n",
    "    np.save('./data/misc/run_counter.npy',run)\n",
    "    return run\n",
    "# Uncomment the next line to manually set the counter if something goes wrong\n",
    "run = set_counter(39)\n",
    "print('This is run # %i' %run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Generate stream IDs\n",
    "stream_tokens = tls.get_credentials_file()['stream_ids']\n",
    "\n",
    "stream_id1 = dict(token=stream_tokens[-1], maxpoints=600)\n",
    "stream_id2 = dict(token=stream_tokens[-2], maxpoints=600)\n",
    "stream_id3 = dict(token=stream_tokens[-3], maxpoints=600)\n",
    "stream_id4 = dict(token=stream_tokens[-4], maxpoints=600)        \n",
    "\n",
    "# Define traces\n",
    "loss = go.Scatter(x=[], y=[], name='Loss', stream=stream_id1, mode='lines+markers',yaxis='y', line=dict(shape='spline',smoothing=1))\n",
    "jaccard = go.Scatter(x=[], y=[], name='Jaccard', stream=stream_id2, mode='lines+markers',yaxis='y2', line=dict(shape='spline',smoothing=1))\n",
    "val_loss = go.Scatter(x=[], y=[], name='Val Loss', stream=stream_id3, mode='lines+markers',yaxis='y', line=dict(shape='spline',smoothing=1))\n",
    "val_jaccard = go.Scatter(x=[], y=[], name='Val Jaccard', stream=stream_id4, mode='lines+markers',yaxis='y2', line=dict(shape='spline',smoothing=1))\n",
    "\n",
    "# Make data and layout objectes\n",
    "data = go.Data([loss, jaccard, val_loss, val_jaccard])\n",
    "layout = go.Layout(title='Loss/Jaccard for run {}'.format(run),\n",
    "                   yaxis=dict(title='Loss',type='log',autorange=True,rangemode='tozero'),\n",
    "                   yaxis2=dict(title='Jaccard index',overlaying='y',side='right',range=[0,1]))\n",
    "\n",
    "# Send plot to plotly, initialize streaming\n",
    "plotly_fig = go.Figure(data=data, layout=layout)\n",
    "\n",
    "# Make stream objects\n",
    "loss_stream = py.Stream(stream_id=stream_tokens[-1])\n",
    "jaccard_stream = py.Stream(stream_id=stream_tokens[-2])\n",
    "val_loss_stream = py.Stream(stream_id=stream_tokens[-3])\n",
    "val_jaccard_stream = py.Stream(stream_id=stream_tokens[-4])\n",
    "\n",
    "\n",
    "training = threading.Event()\n",
    "\n",
    "def heartbeater(training):\n",
    "    while not training.isSet():\n",
    "        loss_stream.heartbeat()\n",
    "        jaccard_stream.heartbeat()\n",
    "        val_loss_stream.heartbeat()\n",
    "        val_jaccard_stream.heartbeat()\n",
    "        time.sleep(5)\n",
    "        \n",
    "t = threading.Thread(target=heartbeater,args=(training,))\n",
    "t.setDaemon(True)\n",
    "\n",
    "    \n",
    "# Define class for streamer callback\n",
    "class streamer(keras.callbacks.Callback):\n",
    "    def on_train_begin(self, logs={}):\n",
    "        \n",
    "        # Open streams\n",
    "        loss_stream.open()\n",
    "        jaccard_stream.open()\n",
    "        val_loss_stream.open()\n",
    "        val_jaccard_stream.open()\n",
    "        \n",
    "        # Start heartbeater process\n",
    "        t.start()\n",
    "        \n",
    "        # Initialize logs\n",
    "        #self.i = 1\n",
    "        self.x = []\n",
    "        self.losses = []\n",
    "        self.val_losses = []\n",
    "        self.jaccards = []\n",
    "        self.val_jaccards = []\n",
    "        #self.logs = []       \n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        \n",
    "        #self.logs.append(logs)\n",
    "        self.x.append(epoch) # Note: this was self.i\n",
    "        self.losses.append(logs.get('loss'))\n",
    "        self.val_losses.append(logs.get('val_loss'))\n",
    "        self.jaccards.append(logs.get('jaccard'))\n",
    "        self.val_jaccards.append(logs.get('val_jaccard'))\n",
    "        #self.i += 1\n",
    "        \n",
    "        loss_stream.write(dict(x=self.x, y=self.losses))\n",
    "        jaccard_stream.write(dict(x=self.x, y=self.jaccards))\n",
    "        val_loss_stream.write(dict(x=self.x, y=self.val_losses))\n",
    "        val_jaccard_stream.write(dict(x=self.x, y=self.val_jaccards))\n",
    "        \n",
    "    def on_train_end(self,logs={}):\n",
    "        \n",
    "        #self.logs.append(logs)\n",
    "        #self.x.append(epoch) # potentially switch to self.epoch?\n",
    "        #self.losses.append(logs.get('loss'))\n",
    "        #self.val_losses.append(logs.get('val_loss'))\n",
    "        #self.jaccards.append(logs.get('jaccard'))\n",
    "        #self.val_jaccards.append(logs.get('val_jaccard'))\n",
    "        \n",
    "        # Define traces\n",
    "        loss = go.Scatter(x=self.x, y=self.losses, name='Loss', mode='lines+markers',yaxis='y', line=dict(shape='spline',smoothing=1))\n",
    "        jaccard = go.Scatter(x=self.x, y=self.jaccards, name='Jaccard', mode='lines+markers',yaxis='y2', line=dict(shape='spline',smoothing=1))\n",
    "        val_loss = go.Scatter(x=self.x, y=self.val_losses, name='Val Loss', mode='lines+markers',yaxis='y', line=dict(shape='spline',smoothing=1))\n",
    "        val_jaccard = go.Scatter(x=self.x, y=self.val_jaccards, name='Val Jaccard', mode='lines+markers',yaxis='y2', line=dict(shape='spline',smoothing=1))\n",
    "\n",
    "        # Define data and layout\n",
    "        data = go.Data([loss, jaccard, val_loss, val_jaccard])\n",
    "        layout = go.Layout(title='Loss/Jaccard for run {}'.format(run),\n",
    "                           yaxis=dict(title='Loss',type='log',autorange=True,rangemode='tozero'),\n",
    "                           yaxis2=dict(title='Jaccard index',overlaying='y',side='right',range=[0,1]),\n",
    "                           xaxis=dict(title='Epoch'))\n",
    "\n",
    "        # Generate plot\n",
    "        plotly_fig = go.Figure(data=data, layout=layout)\n",
    "        \n",
    "        url = py.plot(plotly_fig, filename='deep_microscopy_run_{}'.format(run))\n",
    "        \n",
    "        # Kill the heartbeater\n",
    "        training.set()\n",
    "        \n",
    "        # Close the streams\n",
    "        loss_stream.close()\n",
    "        jaccard_stream.close()\n",
    "        val_loss_stream.close()\n",
    "        val_jaccard_stream.close()\n",
    "        \n",
    "streamer = streamer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Outline arguments\n",
    "data_gen_args = dict(featurewise_center=False,\n",
    "                     featurewise_std_normalization=False,\n",
    "                     rotation_range=90.,\n",
    "                     width_shift_range=0.1,\n",
    "                     height_shift_range=0.1,\n",
    "                     zoom_range=0.2,\n",
    "                     fill_mode='reflect',\n",
    "                     horizontal_flip=True,\n",
    "                     vertical_flip=True,\n",
    "                     data_format='channels_last')\n",
    "\n",
    "# Initialize generators\n",
    "img_datagen = ImageDataGenerator(**data_gen_args)\n",
    "#msk_datagen = ImageDataGenerator(**data_gen_args)\n",
    "\n",
    "# Make data sources\n",
    "filename = 'mosaic_1'\n",
    "with h5py.File('./data/training-data/{}.hdf5'.format(filename),'r+') as f:\n",
    "    x = f['x'][:,:,:,2:3]\n",
    "    y = f['y_nuclei_weighted'][...]\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(x, y, test_size=0.25, random_state=42)\n",
    "#del x,y\n",
    "\n",
    "# Fit the generators\n",
    "seed=13\n",
    "img_datagen.fit(x_train,augment=True,seed=seed)\n",
    "#msk_datagen.fit(y_train,augment=True,seed=seed)\n",
    "\n",
    "batch_size = 5\n",
    "\n",
    "img_train_generator = img_datagen.flow(x=x_train,batch_size=batch_size,shuffle=False,seed=seed)\n",
    "msk_train_generator = img_datagen.flow(x=y_train,batch_size=batch_size,shuffle=False,seed=seed)\n",
    "\n",
    "img_val_generator = img_datagen.flow(x=x_val,batch_size=batch_size,shuffle=False,seed=seed)\n",
    "msk_val_generator = img_datagen.flow(x=y_val,batch_size=batch_size,shuffle=False,seed=seed)\n",
    "\n",
    "train_generator = zip(img_train_generator, msk_train_generator)\n",
    "val_generator = zip(img_val_generator, msk_val_generator)\n",
    "\n",
    "# Call the flow() method\n",
    "#model.fit_generator(train_generator,\n",
    "#                    steps_per_epoch=196, epochs=100,validation_data='XXX', callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.53 s, sys: 57 ms, total: 3.59 s\n",
      "Wall time: 3.65 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def compiler(img_rows = x_train.shape[1],img_cols = x_train.shape[2],\n",
    "            nfilters = 64, act = 'relu',init = 'he_normal',\n",
    "            lr=0.001,decay=0.0,rate=[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2],transpose=False):\n",
    "    \n",
    "    def jaccard(y_true, y_pred,smooth=1.):\n",
    "        intersection = K.sum(y_true * y_pred)\n",
    "        return (intersection + smooth) / (K.sum(y_true) + K.sum(y_pred) - intersection + smooth)\n",
    "    \n",
    "    def weighted_binary_crossentropy(y_true, y_pred):\n",
    "        weights = y_true\n",
    "        y_true = K.clip(y_true,1,2)*-1+2\n",
    "        return K.mean(weights * K.binary_crossentropy(y_pred, y_true), axis=-1)\n",
    "    \n",
    "    def Conv2DReluBatchNorm(n_filter, filter_size, layer_in, activation, kernel_initializer='he_uniform'):\n",
    "        return BatchNormalization(axis=-1)(Activation(activation=act)((Conv2D(n_filter, filter_size, padding='same',kernel_initializer=init)(layer_in))))\n",
    "        \n",
    "    def up_conv(nfilters,filter_factor,layer_in,kernel_initializer=init,activation=act):\n",
    "        if transpose: # This will perform transposed convolution\n",
    "            return BatchNormalization(axis=-1)(Activation(activation=act)(Conv2D(nfilters*filter_factor, (2, 2), padding='same',kernel_initializer=init)(Conv2DTranspose(nfilters*filter_factor,(2,2),strides=2,padding='valid',kernel_initializer=init,activation='relu',data_format='channels_last')(layer_in))))\n",
    "        else: # This is the original upsampling operation\n",
    "            return BatchNormalization(axis=-1)(Activation(activation=act)(Conv2D(nfilters*filter_factor, (2, 2), padding='same',kernel_initializer=init)(UpSampling2D(size=(2, 2))(layer_in))))\n",
    "        \n",
    "    inputs = Input((img_rows, img_cols,1))\n",
    "    \n",
    "    conv1 = Conv2DReluBatchNorm(nfilters, (3, 3), inputs, activation=act,kernel_initializer=init)\n",
    "    conv1 = Conv2DReluBatchNorm(nfilters, (3, 3), conv1, activation=act,kernel_initializer=init)\n",
    "    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n",
    "    pool1 = Dropout(rate=rate[0])(pool1)\n",
    "\n",
    "    conv2 = Conv2DReluBatchNorm(nfilters*2, (3, 3), pool1, activation=act,kernel_initializer=init)\n",
    "    conv2 = Conv2DReluBatchNorm(nfilters*2, (3, 3), conv2, activation=act,kernel_initializer=init)\n",
    "    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n",
    "    pool2 = Dropout(rate=rate[1])(pool2)\n",
    "\n",
    "    conv3 = Conv2DReluBatchNorm(nfilters*4, (3, 3), pool2, activation=act,kernel_initializer=init)\n",
    "    conv3 = Conv2DReluBatchNorm(nfilters*4, (3, 3), conv3, activation=act,kernel_initializer=init)\n",
    "    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)\n",
    "    pool3 = Dropout(rate=rate[2])(pool3)\n",
    "\n",
    "    conv4 = Conv2DReluBatchNorm(nfilters*8, (3, 3), pool3, activation=act,kernel_initializer=init)\n",
    "    conv4 = Conv2DReluBatchNorm(nfilters*8, (3, 3), conv4, activation=act,kernel_initializer=init)\n",
    "    pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)\n",
    "    pool4 = Dropout(rate=rate[3])(pool4)\n",
    "\n",
    "    conv5 = Conv2DReluBatchNorm(nfilters*16, (3, 3), pool4, activation=act,kernel_initializer=init)\n",
    "    conv5 = Conv2DReluBatchNorm(nfilters*16, (3, 3), conv5, activation=act,kernel_initializer=init)\n",
    "    conv5 = Dropout(rate=rate[4])(conv5)\n",
    "        \n",
    "    up6 = Concatenate(axis=3)([up_conv(nfilters,8,conv5), conv4])\n",
    "    #up6 = merge([up_conv(nfilters,8,conv5), conv4], mode='concat', concat_axis=1)\n",
    "    conv6 = Conv2DReluBatchNorm(nfilters*8, (3, 3), up6, activation=act,kernel_initializer=init)\n",
    "    conv6 = Conv2DReluBatchNorm(nfilters*8, (3, 3), conv6, activation=act,kernel_initializer=init)\n",
    "    conv6 = Dropout(rate=rate[5])(conv6)\n",
    "    \n",
    "    up7 = Concatenate(axis=3)([up_conv(nfilters,4,conv6), conv3])\n",
    "    #up7 = merge([up_conv(nfilters,4,conv6), conv3], mode='concat', concat_axis=1)\n",
    "    conv7 = Conv2DReluBatchNorm(nfilters*4, (3, 3), up7, activation=act,kernel_initializer=init)\n",
    "    conv7 = Conv2DReluBatchNorm(nfilters*4, (3, 3), conv7, activation=act,kernel_initializer=init)\n",
    "    conv7 = Dropout(rate=rate[6])(conv7)\n",
    "    \n",
    "    up8 = Concatenate(axis=3)([up_conv(nfilters,2,conv7), conv2])\n",
    "    #up8 = merge([up_conv(nfilters,2,conv7), conv2], mode='concat', concat_axis=1)\n",
    "    conv8 = Conv2DReluBatchNorm(nfilters*2, (3, 3), up8, activation=act,kernel_initializer=init)\n",
    "    conv8 = Conv2DReluBatchNorm(nfilters*2, (3, 3), conv8, activation=act,kernel_initializer=init)\n",
    "    conv8 = Dropout(rate=rate[7])(conv8)\n",
    "\n",
    "    up9 = Concatenate(axis=3)([up_conv(nfilters,1,conv8), conv1])\n",
    "    #up9 = merge([up_conv(nfilters,1,conv8), conv1], mode='concat', concat_axis=1)\n",
    "    conv9 = Conv2DReluBatchNorm(nfilters, (3, 3), up9, activation=act,kernel_initializer=init)\n",
    "    conv9 = Conv2DReluBatchNorm(nfilters, (3, 3), conv9, activation=act,kernel_initializer=init)\n",
    "    conv9 = Dropout(rate=rate[8])(conv9)\n",
    "    \n",
    "    conv10 = Conv2DReluBatchNorm(1, (1, 1), conv9, activation=act,kernel_initializer=init)\n",
    "    output = Activation(activation='sigmoid')(conv10)\n",
    "    \n",
    "    model = Model(inputs=inputs, outputs=output)\n",
    "    \n",
    "    model.compile(optimizer=Adam(lr=lr,decay=decay), loss=weighted_binary_crossentropy,metrics=[jaccard])\n",
    "\n",
    "    return model\n",
    "\n",
    "rate=[0]*9\n",
    "#rate=[0.1,0.2,0.3,0.4,0.5,0.4,0.3,0.2,0.1] # current version\n",
    "#rate=[0.2,0.3,0.4,0.5,0.5,0.5,0.4,0.3,0.2] # symmetric but more dropout\n",
    "#rate=[0.2,0.2,0.3,0.3,0.4,0.4,0.5,0.5,0.6] # increasing\n",
    "\n",
    "model = compiler(img_rows=x_train.shape[1],img_cols=x_train.shape[2],\n",
    "            nfilters=8,act='relu',init='he_normal',\n",
    "            lr=0.001,rate=rate,transpose=False)\n",
    "#model.save_weights('./data/misc/initial_weights.hdf5')\n",
    "#model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is run # 39\n",
      "Epoch 1/700\n",
      "15/15 [==============================] - 93s - loss: 1.1315 - jaccard: 0.5759 - val_loss: 1.2931 - val_jaccard: 0.6633\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\n",
      "Epoch 2/700\n",
      "15/15 [==============================] - 93s - loss: 0.8587 - jaccard: 0.5290 - val_loss: 1.3467 - val_jaccard: 0.6950\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\n",
      "Epoch 3/700\n",
      " 6/15 [===========>..................] - ETA: 47s - loss: 0.8572 - jaccard: 0.5257\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
     ]
    }
   ],
   "source": [
    "def trainer(model,epochs=1000,fit=True,use_existing=False):\n",
    "    print('This is run # {}'.format(run))\n",
    "    \n",
    "    if use_existing:\n",
    "        model.load_weights('./data/weights/model_weights_run_{}_nuclei.hdf5'.format(run-1))\n",
    "        \n",
    "    if fit:\n",
    "        quitter = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=50, verbose=1, mode='auto')\n",
    "        lrreducer = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=20, verbose=1, mode='auto', epsilon=0.001, cooldown=2, min_lr=0)\n",
    "        model_checkpoint = ModelCheckpoint('./data/weights/model_weights_run_{}_nuclei.hdf5'.format(run), monitor='val_loss', save_best_only=True)\n",
    "        csvlogger = CSVLogger('./data/logs/training_log_run_{}_nuclei.csv'.format(run), separator=',', append=True)\n",
    "        # tensorboard = TensorBoard(log_dir='./data/logs/'+'tensorboard_all-classes-run_{:04d}'.format(run), histogram_freq=0, write_graph=True, write_images=False)\n",
    "        # tensorboard --logdir=data/logs\n",
    "        \n",
    "        # Initialize plot, get url, save url to npy file and push url to pixel\n",
    "        url = py.plot(plotly_fig, filename='deep_microscopy_run_{}'.format(run));\n",
    "        np.save('./data/logs/nuclei_plotly_url_run_{}'.format(run),url)\n",
    "        push_url('Plotly link for run {}'.format(run),str(url+'.embed'))\n",
    "        \n",
    "        model.fit_generator(train_generator,\n",
    "                            steps_per_epoch=15, epochs=700,validation_data=val_generator,\n",
    "                            validation_steps=5,callbacks=[model_checkpoint,csvlogger,streamer],\n",
    "                           verbose=1,initial_epoch=0)\n",
    "            \n",
    "    preds = model.predict(x, verbose=1)\n",
    "    np.save('./data/predictions/predictions_run_{}_nuclei.npy'.format(run), preds)\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = trainer(model,fit=True,use_existing=False)\n",
    "model.save('./data/models/u-net-complete-model-run_{}_nuclei.hdf5'.format(run))\n",
    "push('Training  is done',\n",
    " 'Train loss: %f, train jaccard: %f' %(model.history.history['loss'][-1],model.history.history['jaccard'][-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "push('Training is done','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}