DeepCell_tn_nuclear_K2a_data_augmentation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "330e05e3",
   "metadata": {},
   "source": [
    "**Further train DeepCell Models with K's Data and Make Predictions**<br>\n",
    "Starting from model trained with tn nuclear data, further train the model with resized K's images that is cut into 25 512x512 patches, one image at a time.\n",
    "200 epochs, semanticdatagenerator, resized<br>\n",
    "Optimize data augmentation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a72e9c51",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-13T08:53:18.591250Z",
     "iopub.status.busy": "2022-11-13T08:53:18.590594Z",
     "iopub.status.idle": "2022-11-13T08:53:18.593990Z",
     "shell.execute_reply": "2022-11-13T08:53:18.593424Z"
    }
   },
   "outputs": [],
   "source": [
    "starting_model_path = 'tn1.0_nuclear_20221102.h5' # the model trained with nucleus data from the Tissuenet V1.0 dataset\n",
    "epochs=200 # 20 sec per step "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "90f2183c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-13T08:53:18.602170Z",
     "iopub.status.busy": "2022-11-13T08:53:18.601360Z",
     "iopub.status.idle": "2022-11-13T08:53:25.322227Z",
     "shell.execute_reply": "2022-11-13T08:53:25.323282Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7.1\n",
      "0.11.1\n"
     ]
    }
   ],
   "source": [
    "import syotil\n",
    "\n",
    "import numpy as np\n",
    "from skimage import io\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "from timeit import default_timer\n",
    "import os\n",
    "from skimage import img_as_ubyte\n",
    "\n",
    "import deepcell\n",
    "from deepcell import image_generators\n",
    "from deepcell.utils.train_utils import rate_scheduler, get_callbacks, count_gpus\n",
    "from deepcell_toolbox.utils import resize, tile_image, untile_image\n",
    "from deepcell_toolbox.deep_watershed import deep_watershed\n",
    "from deepcell.losses import weighted_categorical_crossentropy\n",
    "from deepcell.model_zoo.panopticnet import PanopticNet\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "from tensorflow.keras.losses import MSE\n",
    "\n",
    "print(tf.__version__)\n",
    "print(deepcell.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c32457c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-13T08:53:25.343168Z",
     "iopub.status.busy": "2022-11-13T08:53:25.342309Z",
     "iopub.status.idle": "2022-11-13T08:53:31.171243Z",
     "shell.execute_reply": "2022-11-13T08:53:31.171602Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-21 17:07:21.517472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 14657 MB memory:  -> device: 0, name: Tesla V100-PCIE-16GB, pci bus id: 0000:04:00.0, compute capability: 7.0\n",
      "2022-11-21 17:07:21.561819: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:1 with 14657 MB memory:  -> device: 1, name: Tesla V100-PCIE-16GB, pci bus id: 0000:06:00.0, compute capability: 7.0\n",
      "2022-11-21 17:07:21.564010: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:2 with 14657 MB memory:  -> device: 2, name: Tesla V100-PCIE-16GB, pci bus id: 0000:08:00.0, compute capability: 7.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7, 1040, 1159, 1)\n",
      "(7, 1040, 1159, 1)\n",
      "2.0\n",
      "(7, 2080, 2318, 1)\n",
      "(7, 2080, 2318, 1)\n",
      "(7, 2080, 2318, 1)\n",
      "(7, 2080, 2318, 1)\n",
      "[0, 400, 800, 1200, 1568]\n",
      "[512, 912, 1312, 1712, 2080]\n",
      "[0, 512, 1024, 1536, 1806]\n",
      "[512, 1024, 1536, 2048, 2318]\n",
      "(175, 512, 512, 1)\n",
      "(175, 512, 512, 1)\n"
     ]
    }
   ],
   "source": [
    "INPUT_PATH=\"images/training7/\"\n",
    "import glob\n",
    "FILENAMES = glob.glob(INPUT_PATH+\"*_img.png\")\n",
    "print(len(FILENAMES))\n",
    "\n",
    "imgs = [io.imread(CURR_IM_NAME)[:,:,0] for CURR_IM_NAME in FILENAMES]\n",
    "X_train = tf.stack(imgs)\n",
    "X_train = np.expand_dims(X_train, axis=-1)\n",
    "print(X_train.shape)\n",
    "\n",
    "masks = [io.imread(CURR_IM_NAME.replace(\"img\",\"masks\")) for CURR_IM_NAME in FILENAMES]\n",
    "y_train = tf.stack(masks)\n",
    "y_train = np.expand_dims(y_train, axis=-1)\n",
    "print(y_train.shape)\n",
    "\n",
    "# resize image\n",
    "mpp_select=1.3 # based on DeepCell_tn_nuclear.ipynb, the best value is 1.2, but 1.3 makes scale factor an integer\n",
    "shape = X_train.shape\n",
    "scale_factor = mpp_select / 0.65\n",
    "print(scale_factor)\n",
    "new_shape = (int(shape[1] * scale_factor),\n",
    "             int(shape[2] * scale_factor))\n",
    "\n",
    "X_train_resized = resize(X_train, new_shape, data_format='channels_last')\n",
    "y_train_resized = resize(y_train, new_shape, data_format='channels_last', labeled_image=True)\n",
    "print(X_train_resized.shape)\n",
    "print(y_train_resized.shape)\n",
    "\n",
    "# cut each resized image into 5x5 patches of size 512x512 \n",
    "print(X_train_resized.shape)\n",
    "print(y_train_resized.shape)\n",
    "\n",
    "x_starts=[0,400,400*2,400*3,X_train_resized.shape[1]-512]\n",
    "x_ends=[x+512 for x in x_starts]\n",
    "print(x_starts)\n",
    "print(x_ends)\n",
    "\n",
    "y_starts=[0,512,512*2,512*3,X_train_resized.shape[2]-512]\n",
    "y_ends=[x+512 for x in y_starts]\n",
    "print(y_starts)\n",
    "print(y_ends)\n",
    "\n",
    "sub_X=tf.concat([tf.concat([X_train_resized[:, x_starts[i]:x_ends[i], y_starts[j]:y_ends[j], :] for i in range(len(x_starts)) ], axis=0) for j in range(len(y_starts))], axis=0)\n",
    "sub_X = np.array(sub_X, dtype=np.uint8) # tf.stack changes the dtype \n",
    "print(sub_X.shape)\n",
    "sub_y=tf.concat([tf.concat([y_train_resized[:, x_starts[i]:x_ends[i], y_starts[j]:y_ends[j], :] for i in range(len(x_starts)) ], axis=0) for j in range(len(y_starts))], axis=0)\n",
    "sub_y = np.array(sub_y, dtype=np.uint16) # tf.stack changes the dtype \n",
    "print(sub_y.shape)\n",
    "\n",
    "# create dataloader\n",
    "seed=0\n",
    "min_objects = 5\n",
    "\n",
    "transforms = ['inner-distance', 'outer-distance', 'fgbg']\n",
    "transforms_kwargs = {'outer-distance': {'erosion_width': 0}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1bac06f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-13T08:53:31.188523Z",
     "iopub.status.busy": "2022-11-13T08:53:31.187837Z",
     "iopub.status.idle": "2022-11-13T15:39:12.876368Z",
     "shell.execute_reply": "2022-11-13T15:39:12.877349Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Removing 6 of 175 images with fewer than 5 objects.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-21 17:07:25.149568: W tensorflow/python/util/util.cc:368] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n",
      "/app/software/TensorFlow/2.7.1-foss-2021b-CUDA-11.4.1/lib/python3.9/site-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on 3 GPUs.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-21 17:07:35.614242: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /device:GPU:0 with 14657 MB memory:  -> device: 0, name: Tesla V100-PCIE-16GB, pci bus id: 0000:04:00.0, compute capability: 7.0\n",
      "2022-11-21 17:07:35.615321: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /device:GPU:1 with 14657 MB memory:  -> device: 1, name: Tesla V100-PCIE-16GB, pci bus id: 0000:06:00.0, compute capability: 7.0\n",
      "2022-11-21 17:07:35.616360: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /device:GPU:2 with 14657 MB memory:  -> device: 2, name: Tesla V100-PCIE-16GB, pci bus id: 0000:08:00.0, compute capability: 7.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-21 17:08:00.640602: I tensorflow/stream_executor/cuda/cuda_dnn.cc:366] Loaded cuDNN version 8202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42/42 [==============================] - ETA: 0s - loss: 0.0163 - semantic_0_loss: 0.0041 - semantic_1_loss: 0.0102 - semantic_2_loss: 0.0020WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 64s 443ms/step - loss: 0.0163 - semantic_0_loss: 0.0041 - semantic_1_loss: 0.0102 - semantic_2_loss: 0.0020 - lr: 1.0000e-04\n",
      "Epoch 2/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0137 - semantic_0_loss: 0.0035 - semantic_1_loss: 0.0088 - semantic_2_loss: 0.0015WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0137 - semantic_0_loss: 0.0035 - semantic_1_loss: 0.0088 - semantic_2_loss: 0.0015 - lr: 9.9000e-05\n",
      "Epoch 3/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0133 - semantic_0_loss: 0.0034 - semantic_1_loss: 0.0085 - semantic_2_loss: 0.0014WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0133 - semantic_0_loss: 0.0034 - semantic_1_loss: 0.0085 - semantic_2_loss: 0.0014 - lr: 9.8010e-05\n",
      "Epoch 4/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0126 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0080 - semantic_2_loss: 0.0014WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0126 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0080 - semantic_2_loss: 0.0014 - lr: 9.7030e-05\n",
      "Epoch 5/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0127 - semantic_0_loss: 0.0033 - semantic_1_loss: 0.0081 - semantic_2_loss: 0.0014WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0127 - semantic_0_loss: 0.0033 - semantic_1_loss: 0.0081 - semantic_2_loss: 0.0014 - lr: 9.6060e-05\n",
      "Epoch 6/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0123 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0078 - semantic_2_loss: 0.0013WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0123 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0078 - semantic_2_loss: 0.0013 - lr: 9.5099e-05\n",
      "Epoch 7/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0124 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0079 - semantic_2_loss: 0.0013WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0124 - semantic_0_loss: 0.0032 - semantic_1_loss: 0.0079 - semantic_2_loss: 0.0013 - lr: 9.4148e-05\n",
      "Epoch 8/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0118 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0075 - semantic_2_loss: 0.0013WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0118 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0075 - semantic_2_loss: 0.0013 - lr: 9.3207e-05\n",
      "Epoch 9/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0117 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0074 - semantic_2_loss: 0.0013WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 359ms/step - loss: 0.0117 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0074 - semantic_2_loss: 0.0013 - lr: 9.2274e-05\n",
      "Epoch 10/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0115 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0073 - semantic_2_loss: 0.0013WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 348ms/step - loss: 0.0115 - semantic_0_loss: 0.0030 - semantic_1_loss: 0.0073 - semantic_2_loss: 0.0013 - lr: 9.1352e-05\n",
      "Epoch 11/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0111 - semantic_0_loss: 0.0029 - semantic_1_loss: 0.0070 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0111 - semantic_0_loss: 0.0029 - semantic_1_loss: 0.0070 - semantic_2_loss: 0.0012 - lr: 9.0438e-05\n",
      "Epoch 12/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0109 - semantic_0_loss: 0.0029 - semantic_1_loss: 0.0069 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0109 - semantic_0_loss: 0.0029 - semantic_1_loss: 0.0069 - semantic_2_loss: 0.0012 - lr: 8.9534e-05\n",
      "Epoch 13/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0109 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0068 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 328ms/step - loss: 0.0109 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0068 - semantic_2_loss: 0.0012 - lr: 8.8638e-05\n",
      "Epoch 14/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0106 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0066 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0106 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0066 - semantic_2_loss: 0.0012 - lr: 8.7752e-05\n",
      "Epoch 15/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0106 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0066 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 332ms/step - loss: 0.0106 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0066 - semantic_2_loss: 0.0012 - lr: 8.6875e-05\n",
      "Epoch 16/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0102 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0063 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0102 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0063 - semantic_2_loss: 0.0012 - lr: 8.6006e-05\n",
      "Epoch 17/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0102 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0063 - semantic_2_loss: 0.0012WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0102 - semantic_0_loss: 0.0028 - semantic_1_loss: 0.0063 - semantic_2_loss: 0.0012 - lr: 8.5146e-05\n",
      "Epoch 18/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0099 - semantic_0_loss: 0.0027 - semantic_1_loss: 0.0061 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0099 - semantic_0_loss: 0.0027 - semantic_1_loss: 0.0061 - semantic_2_loss: 0.0011 - lr: 8.4294e-05\n",
      "Epoch 19/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0096 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0059 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0096 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0059 - semantic_2_loss: 0.0011 - lr: 8.3451e-05\n",
      "Epoch 20/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0095 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0058 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0095 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0058 - semantic_2_loss: 0.0011 - lr: 8.2617e-05\n",
      "Epoch 21/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0095 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0058 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0095 - semantic_0_loss: 0.0026 - semantic_1_loss: 0.0058 - semantic_2_loss: 0.0011 - lr: 8.1791e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0092 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0056 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0092 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0056 - semantic_2_loss: 0.0011 - lr: 8.0973e-05\n",
      "Epoch 23/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0092 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0056 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0092 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0056 - semantic_2_loss: 0.0011 - lr: 8.0163e-05\n",
      "Epoch 24/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0089 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0054 - semantic_2_loss: 0.0011WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0089 - semantic_0_loss: 0.0025 - semantic_1_loss: 0.0054 - semantic_2_loss: 0.0011 - lr: 7.9361e-05\n",
      "Epoch 25/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0087 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0053 - semantic_2_loss: 0.0010WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0087 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0053 - semantic_2_loss: 0.0010 - lr: 7.8568e-05\n",
      "Epoch 26/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0087 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0052 - semantic_2_loss: 0.0010WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0087 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0052 - semantic_2_loss: 0.0010 - lr: 7.7782e-05\n",
      "Epoch 27/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0084 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0050 - semantic_2_loss: 0.0010WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0084 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0050 - semantic_2_loss: 0.0010 - lr: 7.7004e-05\n",
      "Epoch 28/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0084 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0050 - semantic_2_loss: 0.0010WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 348ms/step - loss: 0.0084 - semantic_0_loss: 0.0024 - semantic_1_loss: 0.0050 - semantic_2_loss: 0.0010 - lr: 7.6234e-05\n",
      "Epoch 29/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0082 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0049 - semantic_2_loss: 9.9808e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 344ms/step - loss: 0.0082 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0049 - semantic_2_loss: 9.9808e-04 - lr: 7.5472e-05\n",
      "Epoch 30/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0079 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0047 - semantic_2_loss: 9.7783e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0079 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0047 - semantic_2_loss: 9.7783e-04 - lr: 7.4717e-05\n",
      "Epoch 31/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0080 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0047 - semantic_2_loss: 9.8367e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 340ms/step - loss: 0.0080 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0047 - semantic_2_loss: 9.8367e-04 - lr: 7.3970e-05\n",
      "Epoch 32/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0079 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0046 - semantic_2_loss: 9.7317e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0079 - semantic_0_loss: 0.0023 - semantic_1_loss: 0.0046 - semantic_2_loss: 9.7317e-04 - lr: 7.3230e-05\n",
      "Epoch 33/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0075 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0044 - semantic_2_loss: 9.4958e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0075 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0044 - semantic_2_loss: 9.4958e-04 - lr: 7.2498e-05\n",
      "Epoch 34/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0076 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0045 - semantic_2_loss: 9.6155e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0076 - semantic_0_loss: 0.0022 - semantic_1_loss: 0.0045 - semantic_2_loss: 9.6155e-04 - lr: 7.1773e-05\n",
      "Epoch 35/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0074 - semantic_0_loss: 0.0021 - semantic_1_loss: 0.0043 - semantic_2_loss: 9.4712e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0074 - semantic_0_loss: 0.0021 - semantic_1_loss: 0.0043 - semantic_2_loss: 9.4712e-04 - lr: 7.1055e-05\n",
      "Epoch 36/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0072 - semantic_0_loss: 0.0021 - semantic_1_loss: 0.0042 - semantic_2_loss: 9.2662e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0072 - semantic_0_loss: 0.0021 - semantic_1_loss: 0.0042 - semantic_2_loss: 9.2662e-04 - lr: 7.0345e-05\n",
      "Epoch 37/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0071 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2036e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0071 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2036e-04 - lr: 6.9641e-05\n",
      "Epoch 38/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2169e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2169e-04 - lr: 6.8945e-05\n",
      "Epoch 39/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0040 - semantic_2_loss: 9.2132e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0040 - semantic_2_loss: 9.2132e-04 - lr: 6.8255e-05\n",
      "Epoch 40/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2104e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 353ms/step - loss: 0.0070 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0041 - semantic_2_loss: 9.2104e-04 - lr: 6.7573e-05\n",
      "Epoch 41/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0068 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0039 - semantic_2_loss: 9.0872e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0068 - semantic_0_loss: 0.0020 - semantic_1_loss: 0.0039 - semantic_2_loss: 9.0872e-04 - lr: 6.6897e-05\n",
      "Epoch 42/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0066 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0038 - semantic_2_loss: 8.9407e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0066 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0038 - semantic_2_loss: 8.9407e-04 - lr: 6.6228e-05\n",
      "Epoch 43/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.7457e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.7457e-04 - lr: 6.5566e-05\n",
      "Epoch 44/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0065 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0037 - semantic_2_loss: 8.8371e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0065 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0037 - semantic_2_loss: 8.8371e-04 - lr: 6.4910e-05\n",
      "Epoch 45/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.7835e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.7835e-04 - lr: 6.4261e-05\n",
      "Epoch 46/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.6733e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0064 - semantic_0_loss: 0.0019 - semantic_1_loss: 0.0036 - semantic_2_loss: 8.6733e-04 - lr: 6.3619e-05\n",
      "Epoch 47/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0062 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0035 - semantic_2_loss: 8.7250e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0062 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0035 - semantic_2_loss: 8.7250e-04 - lr: 6.2982e-05\n",
      "Epoch 48/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0060 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.5191e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0060 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.5191e-04 - lr: 6.2353e-05\n",
      "Epoch 49/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0061 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.5908e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0061 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.5908e-04 - lr: 6.1729e-05\n",
      "Epoch 50/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0060 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.4454e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0060 - semantic_0_loss: 0.0018 - semantic_1_loss: 0.0034 - semantic_2_loss: 8.4454e-04 - lr: 6.1112e-05\n",
      "Epoch 51/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0058 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0033 - semantic_2_loss: 8.4021e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0058 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0033 - semantic_2_loss: 8.4021e-04 - lr: 6.0501e-05\n",
      "Epoch 52/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0056 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0032 - semantic_2_loss: 8.2322e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0056 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0032 - semantic_2_loss: 8.2322e-04 - lr: 5.9896e-05\n",
      "Epoch 53/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0056 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2387e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0056 - semantic_0_loss: 0.0017 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2387e-04 - lr: 5.9297e-05\n",
      "Epoch 54/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0056 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2390e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0056 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2390e-04 - lr: 5.8704e-05\n",
      "Epoch 55/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0056 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2537e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0056 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0031 - semantic_2_loss: 8.2537e-04 - lr: 5.8117e-05\n",
      "Epoch 56/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0054 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 8.0815e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 330ms/step - loss: 0.0054 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 8.0815e-04 - lr: 5.7535e-05\n",
      "Epoch 57/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0053 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9741e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0053 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9741e-04 - lr: 5.6960e-05\n",
      "Epoch 58/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0054 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 8.1214e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0054 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 8.1214e-04 - lr: 5.6391e-05\n",
      "Epoch 59/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0053 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 7.9973e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0053 - semantic_0_loss: 0.0016 - semantic_1_loss: 0.0030 - semantic_2_loss: 7.9973e-04 - lr: 5.5827e-05\n",
      "Epoch 60/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9889e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9889e-04 - lr: 5.5268e-05\n",
      "Epoch 61/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9298e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 332ms/step - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0029 - semantic_2_loss: 7.9298e-04 - lr: 5.4716e-05\n",
      "Epoch 62/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42/42 [==============================] - ETA: 0s - loss: 0.0051 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.8616e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0051 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.8616e-04 - lr: 5.4169e-05\n",
      "Epoch 63/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.9312e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 339ms/step - loss: 0.0052 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.9312e-04 - lr: 5.3627e-05\n",
      "Epoch 64/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0050 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0027 - semantic_2_loss: 7.7894e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0050 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0027 - semantic_2_loss: 7.7894e-04 - lr: 5.3091e-05\n",
      "Epoch 65/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0051 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.8631e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 326ms/step - loss: 0.0051 - semantic_0_loss: 0.0015 - semantic_1_loss: 0.0028 - semantic_2_loss: 7.8631e-04 - lr: 5.2560e-05\n",
      "Epoch 66/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0048 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.6870e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0048 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.6870e-04 - lr: 5.2034e-05\n",
      "Epoch 67/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.5836e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.5836e-04 - lr: 5.1514e-05\n",
      "Epoch 68/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0048 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.6236e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0048 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.6236e-04 - lr: 5.0999e-05\n",
      "Epoch 69/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.5470e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.5470e-04 - lr: 5.0489e-05\n",
      "Epoch 70/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.5784e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0047 - semantic_0_loss: 0.0014 - semantic_1_loss: 0.0026 - semantic_2_loss: 7.5784e-04 - lr: 4.9984e-05\n",
      "Epoch 71/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0046 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.5416e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0046 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.5416e-04 - lr: 4.9484e-05\n",
      "Epoch 72/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0046 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.4829e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0046 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.4829e-04 - lr: 4.8989e-05\n",
      "Epoch 73/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0045 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.4369e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0045 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0025 - semantic_2_loss: 7.4369e-04 - lr: 4.8499e-05\n",
      "Epoch 74/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3613e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3613e-04 - lr: 4.8014e-05\n",
      "Epoch 75/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3625e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3625e-04 - lr: 4.7534e-05\n",
      "Epoch 76/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3694e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0044 - semantic_0_loss: 0.0013 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3694e-04 - lr: 4.7059e-05\n",
      "Epoch 77/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2323e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2323e-04 - lr: 4.6588e-05\n",
      "Epoch 78/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3059e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 329ms/step - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0024 - semantic_2_loss: 7.3059e-04 - lr: 4.6122e-05\n",
      "Epoch 79/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2287e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2287e-04 - lr: 4.5661e-05\n",
      "Epoch 80/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2343e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 350ms/step - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2343e-04 - lr: 4.5204e-05\n",
      "Epoch 81/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2978e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 351ms/step - loss: 0.0043 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2978e-04 - lr: 4.4752e-05\n",
      "Epoch 82/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.1608e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.1608e-04 - lr: 4.4305e-05\n",
      "Epoch 83/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2465e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0042 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0023 - semantic_2_loss: 7.2465e-04 - lr: 4.3862e-05\n",
      "Epoch 84/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0041 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0446e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0041 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0446e-04 - lr: 4.3423e-05\n",
      "Epoch 85/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0108e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 339ms/step - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0108e-04 - lr: 4.2989e-05\n",
      "Epoch 86/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0040 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0624e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0040 - semantic_0_loss: 0.0012 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0624e-04 - lr: 4.2559e-05\n",
      "Epoch 87/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0297e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 7.0297e-04 - lr: 4.2133e-05\n",
      "Epoch 88/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 6.9781e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 6.9781e-04 - lr: 4.1712e-05\n",
      "Epoch 89/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 6.9512e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0040 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0022 - semantic_2_loss: 6.9512e-04 - lr: 4.1295e-05\n",
      "Epoch 90/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0039 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.9195e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0039 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.9195e-04 - lr: 4.0882e-05\n",
      "Epoch 91/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.9307e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.9307e-04 - lr: 4.0473e-05\n",
      "Epoch 92/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.8176e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.8176e-04 - lr: 4.0068e-05\n",
      "Epoch 93/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8096e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8096e-04 - lr: 3.9668e-05\n",
      "Epoch 94/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8699e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8699e-04 - lr: 3.9271e-05\n",
      "Epoch 95/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8708e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0038 - semantic_0_loss: 0.0011 - semantic_1_loss: 0.0021 - semantic_2_loss: 6.8708e-04 - lr: 3.8878e-05\n",
      "Epoch 96/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.7638e-04 WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.7638e-04 - lr: 3.8490e-05\n",
      "Epoch 97/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.8119e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.8119e-04 - lr: 3.8105e-05\n",
      "Epoch 98/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0036 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6877e-04    WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0036 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6877e-04 - lr: 3.7724e-05\n",
      "Epoch 99/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.7267e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 337ms/step - loss: 0.0037 - semantic_0_loss: 0.0010 - semantic_1_loss: 0.0020 - semantic_2_loss: 6.7267e-04 - lr: 3.7346e-05\n",
      "Epoch 100/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0036 - semantic_0_loss: 9.8321e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6606e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0036 - semantic_0_loss: 9.8321e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6606e-04 - lr: 3.6973e-05\n",
      "Epoch 101/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0035 - semantic_0_loss: 9.6378e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6654e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0035 - semantic_0_loss: 9.6378e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6654e-04 - lr: 3.6603e-05\n",
      "Epoch 102/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42/42 [==============================] - ETA: 0s - loss: 0.0036 - semantic_0_loss: 9.9317e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6699e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 349ms/step - loss: 0.0036 - semantic_0_loss: 9.9317e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6699e-04 - lr: 3.6237e-05\n",
      "Epoch 103/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0036 - semantic_0_loss: 9.8924e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6486e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0036 - semantic_0_loss: 9.8924e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6486e-04 - lr: 3.5875e-05\n",
      "Epoch 104/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.3412e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5370e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 340ms/step - loss: 0.0034 - semantic_0_loss: 9.3412e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5370e-04 - lr: 3.5516e-05\n",
      "Epoch 105/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0036 - semantic_0_loss: 9.7288e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6767e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0036 - semantic_0_loss: 9.7288e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6767e-04 - lr: 3.5161e-05\n",
      "Epoch 106/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0035 - semantic_0_loss: 9.6234e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6037e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0035 - semantic_0_loss: 9.6234e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.6037e-04 - lr: 3.4809e-05\n",
      "Epoch 107/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0035 - semantic_0_loss: 9.6106e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5932e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0035 - semantic_0_loss: 9.6106e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5932e-04 - lr: 3.4461e-05\n",
      "Epoch 108/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0035 - semantic_0_loss: 9.3807e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5502e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0035 - semantic_0_loss: 9.3807e-04 - semantic_1_loss: 0.0019 - semantic_2_loss: 6.5502e-04 - lr: 3.4117e-05\n",
      "Epoch 109/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.1036e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4663e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0034 - semantic_0_loss: 9.1036e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4663e-04 - lr: 3.3775e-05\n",
      "Epoch 110/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.3415e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.5184e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0034 - semantic_0_loss: 9.3415e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.5184e-04 - lr: 3.3438e-05\n",
      "Epoch 111/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.2242e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.5046e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0034 - semantic_0_loss: 9.2242e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.5046e-04 - lr: 3.3103e-05\n",
      "Epoch 112/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.1904e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4518e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0034 - semantic_0_loss: 9.1904e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4518e-04 - lr: 3.2772e-05\n",
      "Epoch 113/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0034 - semantic_0_loss: 9.0988e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4725e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 342ms/step - loss: 0.0034 - semantic_0_loss: 9.0988e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4725e-04 - lr: 3.2445e-05\n",
      "Epoch 114/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0033 - semantic_0_loss: 8.8654e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4335e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0033 - semantic_0_loss: 8.8654e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.4335e-04 - lr: 3.2120e-05\n",
      "Epoch 115/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0033 - semantic_0_loss: 8.8032e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3584e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0033 - semantic_0_loss: 8.8032e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3584e-04 - lr: 3.1799e-05\n",
      "Epoch 116/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.6943e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3207e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0032 - semantic_0_loss: 8.6943e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3207e-04 - lr: 3.1481e-05\n",
      "Epoch 117/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0033 - semantic_0_loss: 8.7046e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.3422e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0033 - semantic_0_loss: 8.7046e-04 - semantic_1_loss: 0.0018 - semantic_2_loss: 6.3422e-04 - lr: 3.1166e-05\n",
      "Epoch 118/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.6489e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3502e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0032 - semantic_0_loss: 8.6489e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3502e-04 - lr: 3.0854e-05\n",
      "Epoch 119/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.4906e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3332e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0032 - semantic_0_loss: 8.4906e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3332e-04 - lr: 3.0546e-05\n",
      "Epoch 120/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.4318e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3115e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0032 - semantic_0_loss: 8.4318e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3115e-04 - lr: 3.0240e-05\n",
      "Epoch 121/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.4997e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3132e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 346ms/step - loss: 0.0032 - semantic_0_loss: 8.4997e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3132e-04 - lr: 2.9938e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 122/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 8.1214e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.2178e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0031 - semantic_0_loss: 8.1214e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.2178e-04 - lr: 2.9639e-05\n",
      "Epoch 123/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.5599e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3326e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0032 - semantic_0_loss: 8.5599e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3326e-04 - lr: 2.9342e-05\n",
      "Epoch 124/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0032 - semantic_0_loss: 8.5557e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3163e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0032 - semantic_0_loss: 8.5557e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.3163e-04 - lr: 2.9049e-05\n",
      "Epoch 125/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 8.0910e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.2341e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 329ms/step - loss: 0.0031 - semantic_0_loss: 8.0910e-04 - semantic_1_loss: 0.0017 - semantic_2_loss: 6.2341e-04 - lr: 2.8758e-05\n",
      "Epoch 126/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 7.9889e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1937e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 328ms/step - loss: 0.0031 - semantic_0_loss: 7.9889e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1937e-04 - lr: 2.8471e-05\n",
      "Epoch 127/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0030 - semantic_0_loss: 7.8616e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1819e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0030 - semantic_0_loss: 7.8616e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1819e-04 - lr: 2.8186e-05\n",
      "Epoch 128/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 7.9780e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1955e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0031 - semantic_0_loss: 7.9780e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1955e-04 - lr: 2.7904e-05\n",
      "Epoch 129/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 7.9468e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.2002e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0031 - semantic_0_loss: 7.9468e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.2002e-04 - lr: 2.7625e-05\n",
      "Epoch 130/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0030 - semantic_0_loss: 7.8207e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1589e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0030 - semantic_0_loss: 7.8207e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1589e-04 - lr: 2.7349e-05\n",
      "Epoch 131/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 7.9736e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.2071e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0031 - semantic_0_loss: 7.9736e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.2071e-04 - lr: 2.7075e-05\n",
      "Epoch 132/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.6630e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0679e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0029 - semantic_0_loss: 7.6630e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0679e-04 - lr: 2.6805e-05\n",
      "Epoch 133/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0031 - semantic_0_loss: 7.9601e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1830e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0031 - semantic_0_loss: 7.9601e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1830e-04 - lr: 2.6537e-05\n",
      "Epoch 134/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0030 - semantic_0_loss: 7.8681e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1744e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0030 - semantic_0_loss: 7.8681e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1744e-04 - lr: 2.6271e-05\n",
      "Epoch 135/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.4836e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0776e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0029 - semantic_0_loss: 7.4836e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0776e-04 - lr: 2.6009e-05\n",
      "Epoch 136/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0030 - semantic_0_loss: 7.7394e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1421e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 332ms/step - loss: 0.0030 - semantic_0_loss: 7.7394e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1421e-04 - lr: 2.5748e-05\n",
      "Epoch 137/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.5830e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0689e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 332ms/step - loss: 0.0029 - semantic_0_loss: 7.5830e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0689e-04 - lr: 2.5491e-05\n",
      "Epoch 138/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.5531e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0736e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0029 - semantic_0_loss: 7.5531e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0736e-04 - lr: 2.5236e-05\n",
      "Epoch 139/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0030 - semantic_0_loss: 7.6056e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1237e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0030 - semantic_0_loss: 7.6056e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.1237e-04 - lr: 2.4984e-05\n",
      "Epoch 140/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.2692e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9923e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0029 - semantic_0_loss: 7.2692e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9923e-04 - lr: 2.4734e-05\n",
      "Epoch 141/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.5007e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0477e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0029 - semantic_0_loss: 7.5007e-04 - semantic_1_loss: 0.0016 - semantic_2_loss: 6.0477e-04 - lr: 2.4487e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 142/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.2683e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0092e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0029 - semantic_0_loss: 7.2683e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0092e-04 - lr: 2.4242e-05\n",
      "Epoch 143/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.2619e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9722e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0028 - semantic_0_loss: 7.2619e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9722e-04 - lr: 2.3999e-05\n",
      "Epoch 144/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.3841e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9654e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0029 - semantic_0_loss: 7.3841e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9654e-04 - lr: 2.3759e-05\n",
      "Epoch 145/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.0661e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9528e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0028 - semantic_0_loss: 7.0661e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9528e-04 - lr: 2.3522e-05\n",
      "Epoch 146/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 6.9866e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9264e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0028 - semantic_0_loss: 6.9866e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9264e-04 - lr: 2.3286e-05\n",
      "Epoch 147/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0029 - semantic_0_loss: 7.3614e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0687e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0029 - semantic_0_loss: 7.3614e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0687e-04 - lr: 2.3054e-05\n",
      "Epoch 148/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.2375e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0057e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 345ms/step - loss: 0.0028 - semantic_0_loss: 7.2375e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 6.0057e-04 - lr: 2.2823e-05\n",
      "Epoch 149/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.0276e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9778e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0028 - semantic_0_loss: 7.0276e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9778e-04 - lr: 2.2595e-05\n",
      "Epoch 150/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.2386e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9479e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0028 - semantic_0_loss: 7.2386e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9479e-04 - lr: 2.2369e-05\n",
      "Epoch 151/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.8332e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8695e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0027 - semantic_0_loss: 6.8332e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8695e-04 - lr: 2.2145e-05\n",
      "Epoch 152/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.0319e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9155e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0028 - semantic_0_loss: 7.0319e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9155e-04 - lr: 2.1924e-05\n",
      "Epoch 153/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 7.0408e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8905e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0028 - semantic_0_loss: 7.0408e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8905e-04 - lr: 2.1704e-05\n",
      "Epoch 154/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.7652e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8729e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 348ms/step - loss: 0.0027 - semantic_0_loss: 6.7652e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8729e-04 - lr: 2.1487e-05\n",
      "Epoch 155/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 6.8865e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8920e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 350ms/step - loss: 0.0028 - semantic_0_loss: 6.8865e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8920e-04 - lr: 2.1273e-05\n",
      "Epoch 156/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0028 - semantic_0_loss: 6.8558e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9041e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0028 - semantic_0_loss: 6.8558e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.9041e-04 - lr: 2.1060e-05\n",
      "Epoch 157/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.7978e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8432e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 339ms/step - loss: 0.0027 - semantic_0_loss: 6.7978e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8432e-04 - lr: 2.0849e-05\n",
      "Epoch 158/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.5941e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8073e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0027 - semantic_0_loss: 6.5941e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8073e-04 - lr: 2.0641e-05\n",
      "Epoch 159/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.8302e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8983e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0027 - semantic_0_loss: 6.8302e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8983e-04 - lr: 2.0434e-05\n",
      "Epoch 160/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.8070e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8343e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 342ms/step - loss: 0.0027 - semantic_0_loss: 6.8070e-04 - semantic_1_loss: 0.0015 - semantic_2_loss: 5.8343e-04 - lr: 2.0230e-05\n",
      "Epoch 161/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.6458e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7998e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0027 - semantic_0_loss: 6.6458e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7998e-04 - lr: 2.0028e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 162/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.4793e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7862e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0026 - semantic_0_loss: 6.4793e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7862e-04 - lr: 1.9827e-05\n",
      "Epoch 163/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.6075e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8364e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0027 - semantic_0_loss: 6.6075e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8364e-04 - lr: 1.9629e-05\n",
      "Epoch 164/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0027 - semantic_0_loss: 6.5671e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8547e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0027 - semantic_0_loss: 6.5671e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.8547e-04 - lr: 1.9433e-05\n",
      "Epoch 165/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.4388e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7904e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0026 - semantic_0_loss: 6.4388e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7904e-04 - lr: 1.9239e-05\n",
      "Epoch 166/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.5181e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7803e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 340ms/step - loss: 0.0026 - semantic_0_loss: 6.5181e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7803e-04 - lr: 1.9046e-05\n",
      "Epoch 167/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.3504e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7655e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 340ms/step - loss: 0.0026 - semantic_0_loss: 6.3504e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7655e-04 - lr: 1.8856e-05\n",
      "Epoch 168/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.2032e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.6936e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0026 - semantic_0_loss: 6.2032e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.6936e-04 - lr: 1.8667e-05\n",
      "Epoch 169/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.5203e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7792e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 332ms/step - loss: 0.0026 - semantic_0_loss: 6.5203e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7792e-04 - lr: 1.8480e-05\n",
      "Epoch 170/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.3470e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7339e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0026 - semantic_0_loss: 6.3470e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7339e-04 - lr: 1.8296e-05\n",
      "Epoch 171/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.2251e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7094e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0026 - semantic_0_loss: 6.2251e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7094e-04 - lr: 1.8113e-05\n",
      "Epoch 172/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.3078e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7222e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 334ms/step - loss: 0.0026 - semantic_0_loss: 6.3078e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7222e-04 - lr: 1.7932e-05\n",
      "Epoch 173/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.4959e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7461e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0026 - semantic_0_loss: 6.4959e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7461e-04 - lr: 1.7752e-05\n",
      "Epoch 174/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.2917e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7051e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0026 - semantic_0_loss: 6.2917e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7051e-04 - lr: 1.7575e-05\n",
      "Epoch 175/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0564e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6519e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 328ms/step - loss: 0.0025 - semantic_0_loss: 6.0564e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6519e-04 - lr: 1.7399e-05\n",
      "Epoch 176/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.3253e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7292e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0026 - semantic_0_loss: 6.3253e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7292e-04 - lr: 1.7225e-05\n",
      "Epoch 177/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.2275e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7221e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0026 - semantic_0_loss: 6.2275e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7221e-04 - lr: 1.7053e-05\n",
      "Epoch 178/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0548e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6525e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0025 - semantic_0_loss: 6.0548e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6525e-04 - lr: 1.6882e-05\n",
      "Epoch 179/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0750e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6729e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 331ms/step - loss: 0.0025 - semantic_0_loss: 6.0750e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6729e-04 - lr: 1.6713e-05\n",
      "Epoch 180/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0026 - semantic_0_loss: 6.2390e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7140e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 326ms/step - loss: 0.0026 - semantic_0_loss: 6.2390e-04 - semantic_1_loss: 0.0014 - semantic_2_loss: 5.7140e-04 - lr: 1.6546e-05\n",
      "Epoch 181/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.9920e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6302e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0025 - semantic_0_loss: 5.9920e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6302e-04 - lr: 1.6381e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 182/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0044e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6331e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 335ms/step - loss: 0.0025 - semantic_0_loss: 6.0044e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6331e-04 - lr: 1.6217e-05\n",
      "Epoch 183/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0600e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6362e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 339ms/step - loss: 0.0025 - semantic_0_loss: 6.0600e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6362e-04 - lr: 1.6055e-05\n",
      "Epoch 184/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.9540e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6119e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0025 - semantic_0_loss: 5.9540e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6119e-04 - lr: 1.5894e-05\n",
      "Epoch 185/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0122e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6568e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0025 - semantic_0_loss: 6.0122e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6568e-04 - lr: 1.5735e-05\n",
      "Epoch 186/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0171e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6649e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 346ms/step - loss: 0.0025 - semantic_0_loss: 6.0171e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6649e-04 - lr: 1.5578e-05\n",
      "Epoch 187/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.9864e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6266e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 344ms/step - loss: 0.0025 - semantic_0_loss: 5.9864e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6266e-04 - lr: 1.5422e-05\n",
      "Epoch 188/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 6.0059e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6478e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 347ms/step - loss: 0.0025 - semantic_0_loss: 6.0059e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6478e-04 - lr: 1.5268e-05\n",
      "Epoch 189/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.8906e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5678e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 341ms/step - loss: 0.0025 - semantic_0_loss: 5.8906e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5678e-04 - lr: 1.5115e-05\n",
      "Epoch 190/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.8792e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6280e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 338ms/step - loss: 0.0025 - semantic_0_loss: 5.8792e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6280e-04 - lr: 1.4964e-05\n",
      "Epoch 191/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.8269e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5673e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0024 - semantic_0_loss: 5.8269e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5673e-04 - lr: 1.4814e-05\n",
      "Epoch 192/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.7805e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5659e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 343ms/step - loss: 0.0024 - semantic_0_loss: 5.7805e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5659e-04 - lr: 1.4666e-05\n",
      "Epoch 193/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.7504e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5404e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 342ms/step - loss: 0.0024 - semantic_0_loss: 5.7504e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5404e-04 - lr: 1.4520e-05\n",
      "Epoch 194/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.8307e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5979e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0025 - semantic_0_loss: 5.8307e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5979e-04 - lr: 1.4374e-05\n",
      "Epoch 195/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.7976e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6133e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 333ms/step - loss: 0.0025 - semantic_0_loss: 5.7976e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6133e-04 - lr: 1.4231e-05\n",
      "Epoch 196/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.7316e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5498e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 354ms/step - loss: 0.0024 - semantic_0_loss: 5.7316e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5498e-04 - lr: 1.4088e-05\n",
      "Epoch 197/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0025 - semantic_0_loss: 5.8502e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6055e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 15s 341ms/step - loss: 0.0025 - semantic_0_loss: 5.8502e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.6055e-04 - lr: 1.3948e-05\n",
      "Epoch 198/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.7015e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5821e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0024 - semantic_0_loss: 5.7015e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5821e-04 - lr: 1.3808e-05\n",
      "Epoch 199/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.7348e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5714e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 336ms/step - loss: 0.0024 - semantic_0_loss: 5.7348e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5714e-04 - lr: 1.3670e-05\n",
      "Epoch 200/200\n",
      "42/42 [==============================] - ETA: 0s - loss: 0.0024 - semantic_0_loss: 5.6108e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5580e-04WARNING:tensorflow:Can save best model only with val_loss available, skipping.\n",
      "42/42 [==============================] - 14s 337ms/step - loss: 0.0024 - semantic_0_loss: 5.6108e-04 - semantic_1_loss: 0.0013 - semantic_2_loss: 5.5580e-04 - lr: 1.3533e-05\n",
      "/home/yfong/deeplearning/dense_cell_segmentation\n",
      "['M926910_CFL_Position7_CD3_test_img.png', 'M872956_JML_Position10_CD3_test_img.png', 'M872956_JML_Position8_CD8_test_img.png', 'M926910_CFL_Position13_CD3_test_img.png', 'M872956_JML_Position8_CD3_test_img.png', 'M872956_JML_Position8_CD4_test_img.png', 'M872956_JML_Position9_CD3_test_img.png']\n",
      "M926910_CFL_Position7_CD3_test_img.png\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/software/DeepCell/0.11.1-foss-2021b-CUDA-11.4.1/lib/python3.9/site-packages/deepcell_toolbox/deep_watershed.py:179: FutureWarning: `selem` is a deprecated argument name for `h_maxima`. It will be removed in version 1.0.Please use `footprint` instead.\n",
      "  markers = h_maxima(image=maxima,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M872956_JML_Position10_CD3_test_img.png\n",
      "M872956_JML_Position8_CD8_test_img.png\n",
      "M926910_CFL_Position13_CD3_test_img.png\n",
      "M872956_JML_Position8_CD3_test_img.png\n",
      "M872956_JML_Position8_CD4_test_img.png\n",
      "M872956_JML_Position9_CD3_test_img.png\n",
      "[[0.34415584415584416, 0.40215924426450744], [0.2793103448275862, 0.44408427876823336], [0.449438202247191, 0.5049019607843137], [0.2724867724867725, 0.3348115299334812], [0.4144144144144144, 0.5025041736227045], [0.5384615384615384, 0.5772946859903382], [0.3002070393374741, 0.40262172284644193]]\n",
      "                                         0         1         2\n",
      "0   M926910_CFL_Position7_CD3_test_img.png  0.344156  0.402159\n",
      "1  M872956_JML_Position10_CD3_test_img.png   0.27931  0.444084\n",
      "2   M872956_JML_Position8_CD8_test_img.png  0.449438  0.504902\n",
      "3  M926910_CFL_Position13_CD3_test_img.png  0.272487  0.334812\n",
      "4   M872956_JML_Position8_CD3_test_img.png  0.414414  0.502504\n",
      "5   M872956_JML_Position8_CD4_test_img.png  0.538462  0.577295\n",
      "6   M872956_JML_Position9_CD3_test_img.png  0.300207  0.402622\n"
     ]
    }
   ],
   "source": [
    "for ii in [1,2,3,4,5]: \n",
    "    model_name = 'tn1.0_nuclear_K_512x512resized_train7_aug{}'.format(ii)\n",
    "    model_path = '{}.h5'.format(model_name)\n",
    "\n",
    "    # 1. Regular 2. NoScaling 3. NoFlip 4. NoRotation 5. NoScalingRotation \n",
    "    if ii==1:\n",
    "        datagen = image_generators.SemanticDataGenerator(\n",
    "            rotation_range=180,\n",
    "            fill_mode='reflect',\n",
    "            zoom_range=(0.75, 1.25),\n",
    "            horizontal_flip=True,\n",
    "            vertical_flip=True)\n",
    "    elif ii==2:\n",
    "        datagen = image_generators.SemanticDataGenerator(\n",
    "            rotation_range=180,\n",
    "            fill_mode='reflect',\n",
    "            zoom_range=(1, 1),\n",
    "            horizontal_flip=True,\n",
    "            vertical_flip=True)\n",
    "    elif ii==3:\n",
    "        datagen = image_generators.SemanticDataGenerator(\n",
    "            rotation_range=180,\n",
    "            fill_mode='reflect',\n",
    "            zoom_range=(0.75, 1.25),\n",
    "            horizontal_flip=False,\n",
    "            vertical_flip=False)\n",
    "    elif ii==4:\n",
    "        datagen = image_generators.SemanticDataGenerator(\n",
    "            rotation_range=0,\n",
    "            fill_mode='reflect',\n",
    "            zoom_range=(0.75, 1.25),\n",
    "            horizontal_flip=True,\n",
    "            vertical_flip=True)\n",
    "    elif ii==5:\n",
    "        datagen = image_generators.SemanticDataGenerator(\n",
    "            rotation_range=0,\n",
    "            fill_mode='reflect',\n",
    "            zoom_range=(1, 1),\n",
    "            horizontal_flip=True,\n",
    "            vertical_flip=True)\n",
    "\n",
    "    # use augmentation for training but not validation\n",
    "    datagen_val = image_generators.CroppingDataGenerator()\n",
    "\n",
    "    batch_size = 4 # 8 causes memory outage\n",
    "\n",
    "    train_data = datagen.flow(\n",
    "        {'X': sub_X, 'y': sub_y},\n",
    "        seed=seed,\n",
    "        transforms=transforms,\n",
    "        transforms_kwargs=transforms_kwargs,\n",
    "        min_objects=min_objects,\n",
    "        batch_size=batch_size)\n",
    "\n",
    "    val_data = None\n",
    "\n",
    "    semantic_classes = [1, 1, 2] # inner distance, outer distance, fgbg\n",
    "\n",
    "    model = PanopticNet(\n",
    "        backbone='resnet50',\n",
    "        input_shape=(512,512,1),\n",
    "        norm_method='whole_image',\n",
    "        num_semantic_classes=semantic_classes)\n",
    "\n",
    "    lr = 1e-4\n",
    "    optimizer = Adam(lr=lr, clipnorm=0.001)\n",
    "    lr_sched = rate_scheduler(lr=lr, decay=0.99)\n",
    "\n",
    "    # Create a dictionary of losses for each semantic head\n",
    "\n",
    "    def semantic_loss(n_classes):\n",
    "        def _semantic_loss(y_pred, y_true):\n",
    "            if n_classes > 1:\n",
    "                return 0.01 * weighted_categorical_crossentropy(\n",
    "                    y_pred, y_true, n_classes=n_classes)\n",
    "            return MSE(y_pred, y_true)\n",
    "        return _semantic_loss\n",
    "\n",
    "    loss = {}\n",
    "\n",
    "    # Give losses for all of the semantic heads\n",
    "    for layer in model.layers:\n",
    "        if layer.name.startswith('semantic_'):\n",
    "            n_classes = layer.output_shape[-1]\n",
    "            loss[layer.name] = semantic_loss(n_classes)\n",
    "\n",
    "    model.compile(loss=loss, optimizer=optimizer)\n",
    "\n",
    "    model.load_weights(starting_model_path, by_name=True)\n",
    "\n",
    "    [(layer.name, layer.output_shape) for layer in filter(lambda x: x.name.startswith('semantic_'), model.layers)]\n",
    "\n",
    "\n",
    "    # fit the model\n",
    "    print('Training on', count_gpus(), 'GPUs.')\n",
    "\n",
    "    train_callbacks = get_callbacks(\n",
    "        model_path,\n",
    "        lr_sched=lr_sched,\n",
    "    #     monitor=\"val_loss\",\n",
    "        # monitor='loss', # training loss\n",
    "        verbose=1)          \n",
    "\n",
    "    loss_history = model.fit(\n",
    "        train_data,\n",
    "        steps_per_epoch=train_data.y.shape[0] // batch_size, # \n",
    "        epochs=epochs, \n",
    "    #     validation_data=val_data,\n",
    "    #     validation_steps=val_data.y.shape[0] // batch_size,\n",
    "        callbacks=train_callbacks)\n",
    "\n",
    "    model.save_weights(model_path)\n",
    "\n",
    "    prediction_model = PanopticNet(\n",
    "        backbone='resnet50',\n",
    "        norm_method='whole_image',\n",
    "        num_semantic_classes=[1, 1], # inner distance, outer distance\n",
    "        input_shape= (512,512,1)\n",
    "    )\n",
    "\n",
    "    prediction_model.load_weights(model_path, by_name=True)\n",
    "\n",
    "    from deepcell.applications import NuclearSegmentation\n",
    "    app = NuclearSegmentation(prediction_model)\n",
    "    [(layer.name, layer.output_shape) for layer in filter(lambda x: x.name.startswith('semantic_'), app.model.layers)]\n",
    "\n",
    "\n",
    "    import os\n",
    "    print(os.getcwd())\n",
    "    INPUT_PATH=\"images/test_masks/\"\n",
    "    FILENAMES = [f for f in os.listdir(\"images/test_images\")]\n",
    "    print(FILENAMES)\n",
    "\n",
    "\n",
    "    # more detailed mapping of image_mpp parameter\n",
    "    mpps=[1,1.3]\n",
    "\n",
    "    AP_mat=[]\n",
    "    for CURR_IM_NAME in FILENAMES:\n",
    "        print(CURR_IM_NAME)\n",
    "        im0 = io.imread(os.path.join(\"images/test_images\", CURR_IM_NAME))\n",
    "        mask_true=io.imread(os.path.join(INPUT_PATH, CURR_IM_NAME.replace(\"img\",\"masks\")))\n",
    "\n",
    "        x = np.expand_dims(im0, axis=-1)\n",
    "        x = np.expand_dims(x, axis=0)\n",
    "        y, tile_info = app._tile_input(x)\n",
    "        AP_arr=[]\n",
    "        for mpp in mpps:\n",
    "            prd = app._untile_output(app.predict(y, image_mpp=mpp), tile_info)[0,:,:,0]\n",
    "            a=syotil.csi(mask_true, prd)\n",
    "            AP_arr.append(a)\n",
    "        AP_mat.append(AP_arr)\n",
    "\n",
    "    print(AP_mat)\n",
    "\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame([FILENAMES, [x[0] for x in AP_mat], [x[1] for x in AP_mat]])\n",
    "    print(df.transpose())\n",
    "    df.to_csv('APresults/csi_'+model_name+'.txt', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ee97762e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/yfong/deeplearning/dense_cell_segmentation\n",
      "['M926910_CFL_Position7_CD3_test_img.png', 'M872956_JML_Position10_CD3_test_img.png', 'M872956_JML_Position8_CD8_test_img.png', 'M926910_CFL_Position13_CD3_test_img.png', 'M872956_JML_Position8_CD3_test_img.png', 'M872956_JML_Position8_CD4_test_img.png', 'M872956_JML_Position9_CD3_test_img.png']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M926910_CFL_Position7_CD3_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M872956_JML_Position10_CD3_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M872956_JML_Position8_CD8_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M926910_CFL_Position13_CD3_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M872956_JML_Position8_CD3_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M872956_JML_Position8_CD4_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'M926910_CFL_Position7_CD3_test_img.png': 0.4177978883861237, 'M872956_JML_Position10_CD3_test_img.png': 0.43243243243243246, 'M872956_JML_Position8_CD8_test_img.png': 0.5326633165829145, 'M926910_CFL_Position13_CD3_test_img.png': 0.36803874092009686, 'M872956_JML_Position8_CD3_test_img.png': 0.5597147950089126, 'M872956_JML_Position8_CD4_test_img.png': 0.5760598503740648, 'M872956_JML_Position9_CD3_test_img.png': 0.4433198380566802, 'mAP': 0.4757181231087465}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_572/3500599014.py:37: UserWarning: M872956_JML_Position9_CD3_test_dcmasks.png is a low contrast image\n",
      "  io.imsave(CURR_IM_NAME.replace(\"img\",\"dcmasks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n"
     ]
    }
   ],
   "source": [
    "# no rotation  \n",
    "model_name = 'tn1.0_nuclear_K_512x512resized_train7_aug4'\n",
    "model_path = '{}.h5'.format(model_name)\n",
    "\n",
    "prediction_model = PanopticNet(\n",
    "    backbone='resnet50',\n",
    "    norm_method='whole_image',\n",
    "    num_semantic_classes=[1, 1], # inner distance, outer distance\n",
    "    input_shape= (512,512,1)\n",
    ")\n",
    "\n",
    "prediction_model.load_weights(model_path, by_name=True)\n",
    "\n",
    "from deepcell.applications import NuclearSegmentation\n",
    "app = NuclearSegmentation(prediction_model)\n",
    "[(layer.name, layer.output_shape) for layer in filter(lambda x: x.name.startswith('semantic_'), app.model.layers)]\n",
    "\n",
    "\n",
    "import os\n",
    "print(os.getcwd())\n",
    "INPUT_PATH=\"images/test_masks/\"\n",
    "FILENAMES = [f for f in os.listdir(\"images/test_images\")]\n",
    "print(FILENAMES)\n",
    "\n",
    "mpp_select=1.3\n",
    "\n",
    "APs={}\n",
    "for CURR_IM_NAME in FILENAMES:\n",
    "    im0 = io.imread(os.path.join(\"images/test_images\", CURR_IM_NAME))\n",
    "    mask_true=io.imread(os.path.join(INPUT_PATH, CURR_IM_NAME.replace(\"img\",\"masks\")))\n",
    "\n",
    "    x = np.expand_dims(im0, axis=-1)\n",
    "    x = np.expand_dims(x, axis=0)\n",
    "    y, tile_info = app._tile_input(x)\n",
    "    pred = app.predict(y, image_mpp=mpp_select)\n",
    "    prd = app._untile_output(pred, tile_info)\n",
    "    io.imsave(CURR_IM_NAME.replace(\"img\",\"dc_masks\"), np.array(prd[0,:,:,0], dtype=np.uint16))\n",
    "    \n",
    "    APs[CURR_IM_NAME] = syotil.csi(mask_true, prd[0,:,:,0])# masks may lose one pixel if dimension is odd pixels\n",
    "\n",
    "APs[\"mAP\"]=np.mean(list(APs.values()))\n",
    "print(APs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}