img_enhancement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import imageio\n",
    "import cv2\n",
    "import scipy, scipy.misc, scipy.signal\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "CAIP2017 - 17th international Conference on Computer Analysis of Images and Patterns\n",
    " \n",
    "A New Image Contrast Enhancement Algorithm using Exposure Fusion Framework\n",
    "\n",
    "Ying, Zhenqiang & Li, Ge & Ren, Yurui & Wang, Ronggang & Wang, Wenmin. (2017). 36-46. 10.1007/978-3-319-64698-5_4. \n",
    "\"\"\"\n",
    "def computeTextureWeights(fin, sigma, sharpness):\n",
    "    dt0_v = np.vstack((np.diff(fin, n=1, axis=0), fin[0,:]-fin[-1,:]))\n",
    "    dt0_h = np.vstack((np.diff(fin, n=1, axis=1).conj().T, fin[:,0].conj().T-fin[:,-1].conj().T)).conj().T\n",
    "\n",
    "    gauker_h = scipy.signal.convolve2d(dt0_h, np.ones((1,sigma)), mode='same')\n",
    "    gauker_v = scipy.signal.convolve2d(dt0_v, np.ones((sigma,1)), mode='same')\n",
    "\n",
    "    W_h = 1/(np.abs(gauker_h)*np.abs(dt0_h)+sharpness)\n",
    "    W_v = 1/(np.abs(gauker_v)*np.abs(dt0_v)+sharpness)\n",
    "\n",
    "    return  W_h, W_v\n",
    "\n",
    "def solveLinearEquation(IN, wx, wy, lamda):\n",
    "    [r, c] = IN.shape\n",
    "    k = r * c\n",
    "    dx =  -lamda * wx.flatten('F')\n",
    "    dy =  -lamda * wy.flatten('F')\n",
    "    tempx = np.roll(wx, 1, axis=1)\n",
    "    tempy = np.roll(wy, 1, axis=0)\n",
    "    dxa = -lamda *tempx.flatten('F')\n",
    "    dya = -lamda *tempy.flatten('F')\n",
    "    tmp = wx[:,-1]\n",
    "    tempx = np.concatenate((tmp[:,None], np.zeros((r,c-1))), axis=1)\n",
    "    tmp = wy[-1,:]\n",
    "    tempy = np.concatenate((tmp[None,:], np.zeros((r-1,c))), axis=0)\n",
    "    dxd1 = -lamda * tempx.flatten('F')\n",
    "    dyd1 = -lamda * tempy.flatten('F')\n",
    "\n",
    "    wx[:,-1] = 0\n",
    "    wy[-1,:] = 0\n",
    "    dxd2 = -lamda * wx.flatten('F')\n",
    "    dyd2 = -lamda * wy.flatten('F')\n",
    "\n",
    "    Ax = scipy.sparse.spdiags(np.concatenate((dxd1[:,None], dxd2[:,None]), axis=1).T, np.array([-k+r,-r]), k, k)\n",
    "    Ay = scipy.sparse.spdiags(np.concatenate((dyd1[None,:], dyd2[None,:]), axis=0), np.array([-r+1,-1]), k, k)\n",
    "    D = 1 - ( dx + dy + dxa + dya)\n",
    "    A = ((Ax+Ay) + (Ax+Ay).conj().T + scipy.sparse.spdiags(D, 0, k, k)).T\n",
    "\n",
    "    tin = IN[:,:]\n",
    "    tout = scipy.sparse.linalg.spsolve(A, tin.flatten('F'))\n",
    "    OUT = np.reshape(tout, (r, c), order='F')\n",
    "\n",
    "    return OUT\n",
    "\n",
    "\n",
    "def tsmooth(img, lamda=0.01, sigma=3.0, sharpness=0.001):\n",
    "    I = cv2.normalize(img.astype('float64'), None, 0.0, 1.0, cv2.NORM_MINMAX)\n",
    "    x = np.copy(I)\n",
    "    wx, wy = computeTextureWeights(x, sigma, sharpness)\n",
    "    S = solveLinearEquation(I, wx, wy, lamda)\n",
    "    return S\n",
    "\n",
    "def rgb2gm(I):\n",
    "    if (I.shape[2] == 3):\n",
    "        I = cv2.normalize(I.astype('float64'), None, 0.0, 1.0, cv2.NORM_MINMAX)\n",
    "        I = (I[:,:,0]*I[:,:,1]*I[:,:,2])**(1/3)\n",
    "\n",
    "    return I\n",
    "\n",
    "def applyK(I, k, a=-0.3293, b=1.1258):\n",
    "    f = lambda x: np.exp((1-x**a)*b)\n",
    "    beta = f(k)\n",
    "    gamma = k**a\n",
    "    J = (I**gamma)*beta\n",
    "    return J\n",
    "\n",
    "def entropy(X):\n",
    "    tmp = X * 255\n",
    "    tmp[tmp > 255] = 255\n",
    "    tmp[tmp<0] = 0\n",
    "    tmp = tmp.astype(np.uint8)\n",
    "    _, counts = np.unique(tmp, return_counts=True)\n",
    "    pk = np.asarray(counts)\n",
    "    pk = 1.0*pk / np.sum(pk, axis=0)\n",
    "    S = -np.sum(pk * np.log2(pk), axis=0)\n",
    "    return S\n",
    "\n",
    "def maxEntropyEnhance(I, isBad, a=-0.3293, b=1.1258):\n",
    "    # Esatimate k\n",
    "    tmp = cv2.resize(I, (50,50), interpolation=cv2.INTER_AREA)\n",
    "    tmp[tmp<0] = 0\n",
    "    tmp = tmp.real\n",
    "    Y = rgb2gm(tmp)\n",
    "\n",
    "    isBad = isBad * 1\n",
    "    isBad = scipy.misc.imresize(isBad, (50,50), interp='bicubic', mode='F')\n",
    "    isBad[isBad<0.5] = 0\n",
    "    isBad[isBad>=0.5] = 1\n",
    "    Y = Y[isBad==1]\n",
    "\n",
    "    if Y.size == 0:\n",
    "       J = I\n",
    "       return J\n",
    "\n",
    "    f = lambda k: -entropy(applyK(Y, k))\n",
    "    opt_k = scipy.optimize.fminbound(f, 1, 7)\n",
    "\n",
    "    # Apply k\n",
    "    J = applyK(I, opt_k, a, b) - 0.01\n",
    "    return J\n",
    "\n",
    "\n",
    "def Ying_2017_CAIP(img, mu=0.5, a=-0.3293, b=1.1258):\n",
    "    lamda = 0.5\n",
    "    sigma = 5\n",
    "    I = cv2.normalize(img.astype('float64'), None, 0.0, 1.0, cv2.NORM_MINMAX)\n",
    "\n",
    "    # Weight matrix estimation\n",
    "    t_b = np.max(I, axis=2)\n",
    "    t_our = cv2.resize(tsmooth(scipy.misc.imresize(t_b, 0.5, interp='bicubic', mode='F'), lamda, sigma), (t_b.shape[1], t_b.shape[0]), interpolation=cv2.INTER_AREA)\n",
    "\n",
    "    # Apply camera model with k(exposure ratio)\n",
    "    isBad = t_our < 0.5\n",
    "    J = maxEntropyEnhance(I, isBad)\n",
    "\n",
    "    # W: Weight Matrix\n",
    "    t = np.zeros((t_our.shape[0], t_our.shape[1], I.shape[2]))\n",
    "    for i in range(I.shape[2]):\n",
    "        t[:,:,i] = t_our\n",
    "    W = t**mu\n",
    "\n",
    "    I2 = I*W\n",
    "    J2 = J*(1-W)\n",
    "\n",
    "    result = I2 + J2\n",
    "    result = result * 255\n",
    "    result[result > 255] = 255\n",
    "    result[result<0] = 0\n",
    "    return result.astype(np.uint8)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing  2015_02423.jpg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ambersahdev/Envs/CV_final_project/lib/python3.6/site-packages/ipykernel_launcher.py:117: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.3.0.\n",
      "Use Pillow instead: ``numpy.array(Image.fromarray(arr).resize())``.\n",
      "/Users/ambersahdev/Envs/CV_final_project/lib/python3.6/site-packages/ipykernel_launcher.py:93: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.3.0.\n",
      "Use Pillow instead: ``numpy.array(Image.fromarray(arr).resize())``.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing  2015_02430.jpg\n",
      "Processing  2015_02440.jpg\n",
      "Processing  2015_02491.jpg\n",
      "Processing  2015_02413.jpg\n",
      "Processing  2015_02439.jpg\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ambersahdev/Envs/CV_final_project/lib/python3.6/site-packages/ipykernel_launcher.py:63: RuntimeWarning: invalid value encountered in power\n",
      "/Users/ambersahdev/Envs/CV_final_project/lib/python3.6/site-packages/ipykernel_launcher.py:76: RuntimeWarning: invalid value encountered in greater\n",
      "/Users/ambersahdev/Envs/CV_final_project/lib/python3.6/site-packages/ipykernel_launcher.py:77: RuntimeWarning: invalid value encountered in less\n"
     ]
    }
   ],
   "source": [
    "# Image Enhancement using CAIP2017 Algorithm Above\n",
    "DATA_DIR = 'img_enhance_data/'\n",
    "RESULTS_DIR = 'img_enhance_results/'\n",
    "for filename in os.listdir(DATA_DIR):\n",
    "    print(\"Processing \", filename)\n",
    "    src = DATA_DIR + filename\n",
    "    dest = RESULTS_DIR + 'processed_' + filename\n",
    "    img = imageio.imread(src)\n",
    "    result = Ying_2017_CAIP(img)\n",
    "    plt.imsave(dest, result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "CV_final_project",
   "language": "python",
   "name": "cv_final_project"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}