wav2lip_test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "wav2lip_test",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Zju-George/DeepMimic/blob/master/wav2lip_test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vSQFs_G8caeE",
        "colab_type": "text"
      },
      "source": [
        "# Collab preliminaries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XIVB0Xn1g6ih",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qciH4PsUazL_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "d597ad8a-d98f-4f72-85a8-c7b590bb392c"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "4/4AHZjmp4Go7w9d6aFuYXbQMcpzQ3DxDqr5_23xVvhwbx13YiBYfVDno\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yJ5taGmPcWV-",
        "colab_type": "text"
      },
      "source": [
        "# Get the code and models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3LihClHbUd3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "outputId": "d658c34b-f3cf-4083-c721-e9aa6fde9e42"
      },
      "source": [
        "!git clone https://github.com/Rudrabha/Wav2Lip.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Wav2Lip'...\n",
            "remote: Enumerating objects: 146, done.\u001b[K\n",
            "remote: Counting objects: 100% (146/146), done.\u001b[K\n",
            "remote: Compressing objects: 100% (128/128), done.\u001b[K\n",
            "remote: Total 146 (delta 66), reused 63 (delta 11), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (146/146), 60.79 KiB | 787.00 KiB/s, done.\n",
            "Resolving deltas: 100% (66/66), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YjzMPy_Sb0AI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "15730b97-9227-449e-d537-a1a46d670336"
      },
      "source": [
        "!cp -ri \"/content/gdrive/My Drive/Wav2lip/wav2lip_gan.pth\" /content/Wav2Lip/checkpoints/\n",
        "!cd /content/Wav2Lip/checkpoints && ls -l\n",
        "# wav2lip should have 444M size"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 425596\n",
            "-rw-r--r-- 1 root root        46 Sep 10 07:34 README.md\n",
            "-rw------- 1 root root 435801865 Sep 10 07:34 wav2lip_gan.pth\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aWTaOS3ncFt6",
        "colab_type": "text"
      },
      "source": [
        "# Get the pre-requisites"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ooh28vw-Uvd3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "cf26fdf8-1865-4f8c-b676-7f14cbf0c6bd"
      },
      "source": [
        "!pip uninstall tensorflow tensorflow-gpu"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling tensorflow-2.3.0:\n",
            "  Would remove:\n",
            "    /usr/local/bin/estimator_ckpt_converter\n",
            "    /usr/local/bin/saved_model_cli\n",
            "    /usr/local/bin/tensorboard\n",
            "    /usr/local/bin/tf_upgrade_v2\n",
            "    /usr/local/bin/tflite_convert\n",
            "    /usr/local/bin/toco\n",
            "    /usr/local/bin/toco_from_protos\n",
            "    /usr/local/lib/python3.6/dist-packages/tensorflow-2.3.0.dist-info/*\n",
            "    /usr/local/lib/python3.6/dist-packages/tensorflow/*\n",
            "Proceed (y/n)? y\n",
            "  Successfully uninstalled tensorflow-2.3.0\n",
            "\u001b[33mWARNING: Skipping tensorflow-gpu as it is not installed.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "49dCYlLdcK2D",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a12bc29f-7346-4627-c67d-07db50b6debd"
      },
      "source": [
        "!cd /content/Wav2Lip && pip install -r requirements.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting librosa==0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ad/6e/0eb0de1c9c4e02df0b40e56f258eb79bd957be79b918511a184268e01720/librosa-0.7.0.tar.gz (1.6MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6MB 6.5MB/s \n",
            "\u001b[?25hCollecting numpy==1.17.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/92/57179ed45307ec6179e344231c47da7f3f3da9e2eee5c8ab506bd279ce4e/numpy-1.17.1-cp36-cp36m-manylinux1_x86_64.whl (20.4MB)\n",
            "\u001b[K     |████████████████████████████████| 20.4MB 1.5MB/s \n",
            "\u001b[?25hCollecting opencv-contrib-python==4.2.0.34\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c6/db/790dbc6bcfea87fc6f790c6306509c2691ce31c96d82e5b826545d90ea52/opencv_contrib_python-4.2.0.34-cp36-cp36m-manylinux1_x86_64.whl (34.2MB)\n",
            "\u001b[K     |████████████████████████████████| 34.2MB 66kB/s \n",
            "\u001b[?25hCollecting opencv-python==4.1.0.25\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7b/d2/a2dbf83d4553ca6b3701d91d75e42fe50aea97acdc00652dca515749fb5d/opencv_python-4.1.0.25-cp36-cp36m-manylinux1_x86_64.whl (26.6MB)\n",
            "\u001b[K     |████████████████████████████████| 26.6MB 82kB/s \n",
            "\u001b[?25hCollecting tensorflow==1.12.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/22/cc/ca70b78087015d21c5f3f93694107f34ebccb3be9624385a911d4b52ecef/tensorflow-1.12.0-cp36-cp36m-manylinux1_x86_64.whl (83.1MB)\n",
            "\u001b[K     |████████████████████████████████| 83.1MB 37kB/s \n",
            "\u001b[?25hCollecting torch==1.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/60/f685fb2cfb3088736bafbc9bdbb455327bdc8906b606da9c9a81bae1c81e/torch-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (676.9MB)\n",
            "\u001b[K     |████████████████████████████████| 676.9MB 27kB/s \n",
            "\u001b[?25hCollecting torchvision==0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2e/45/0f2f3062c92d9cf1d5d7eabd3cae88cea9affbd2b17fb1c043627838cb0a/torchvision-0.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6MB 46.2MB/s \n",
            "\u001b[?25hCollecting tqdm==4.45.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4a/1c/6359be64e8301b84160f6f6f7936bbfaaa5e9a4eab6cbc681db07600b949/tqdm-4.45.0-py2.py3-none-any.whl (60kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 7.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (2.1.8)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.12 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (0.16.0)\n",
            "Requirement already satisfied: decorator>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (4.4.2)\n",
            "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (1.15.0)\n",
            "Requirement already satisfied: resampy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (0.2.2)\n",
            "Requirement already satisfied: numba>=0.38.0 in /usr/local/lib/python3.6/dist-packages (from librosa==0.7.0->-r requirements.txt (line 1)) (0.48.0)\n",
            "Collecting soundfile>=0.9.0\n",
            "  Downloading https://files.pythonhosted.org/packages/eb/f2/3cbbbf3b96fb9fa91582c438b574cff3f45b29c772f94c400e2c99ef5db9/SoundFile-0.10.3.post1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (1.1.2)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (0.8.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (0.3.3)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (1.1.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (3.12.4)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (1.31.0)\n",
            "Collecting keras-applications>=1.0.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (0.35.1)\n",
            "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0->-r requirements.txt (line 5)) (0.8.1)\n",
            "Collecting tensorboard<1.13.0,>=1.12.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/53/8d32ce9471c18f8d99028b7cef2e5b39ea8765bd7ef250ca05b490880971/tensorboard-1.12.2-py3-none-any.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1MB 40.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision==0.3.0->-r requirements.txt (line 7)) (7.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.0->librosa==0.7.0->-r requirements.txt (line 1)) (49.6.0)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.0->librosa==0.7.0->-r requirements.txt (line 1)) (0.31.0)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.6/dist-packages (from soundfile>=0.9.0->librosa==0.7.0->-r requirements.txt (line 1)) (1.14.2)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.12.0->-r requirements.txt (line 5)) (2.10.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0->-r requirements.txt (line 5)) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0->-r requirements.txt (line 5)) (3.2.2)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi>=1.0->soundfile>=0.9.0->librosa==0.7.0->-r requirements.txt (line 1)) (2.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0->-r requirements.txt (line 5)) (1.7.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0->-r requirements.txt (line 5)) (3.1.0)\n",
            "Building wheels for collected packages: librosa\n",
            "  Building wheel for librosa (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for librosa: filename=librosa-0.7.0-cp36-none-any.whl size=1598345 sha256=0ccd3103e840f1afb87a75c6db635cb4f7ec8bd5187e48711e5bc4863f941c8d\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/1d/38/c8ad12fcad67569d8e730c3275be5e581bd589558484a0f881\n",
            "Successfully built librosa\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numpy, soundfile, librosa, opencv-contrib-python, opencv-python, keras-applications, tensorboard, tensorflow, torch, torchvision, tqdm\n",
            "  Found existing installation: numpy 1.18.5\n",
            "    Uninstalling numpy-1.18.5:\n",
            "      Successfully uninstalled numpy-1.18.5\n",
            "  Found existing installation: librosa 0.6.3\n",
            "    Uninstalling librosa-0.6.3:\n",
            "      Successfully uninstalled librosa-0.6.3\n",
            "  Found existing installation: opencv-contrib-python 4.1.2.30\n",
            "    Uninstalling opencv-contrib-python-4.1.2.30:\n",
            "      Successfully uninstalled opencv-contrib-python-4.1.2.30\n",
            "  Found existing installation: opencv-python 4.1.2.30\n",
            "    Uninstalling opencv-python-4.1.2.30:\n",
            "      Successfully uninstalled opencv-python-4.1.2.30\n",
            "  Found existing installation: tensorboard 2.3.0\n",
            "    Uninstalling tensorboard-2.3.0:\n",
            "      Successfully uninstalled tensorboard-2.3.0\n",
            "  Found existing installation: torch 1.6.0+cu101\n",
            "    Uninstalling torch-1.6.0+cu101:\n",
            "      Successfully uninstalled torch-1.6.0+cu101\n",
            "  Found existing installation: torchvision 0.7.0+cu101\n",
            "    Uninstalling torchvision-0.7.0+cu101:\n",
            "      Successfully uninstalled torchvision-0.7.0+cu101\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "Successfully installed keras-applications-1.0.8 librosa-0.7.0 numpy-1.17.1 opencv-contrib-python-4.2.0.34 opencv-python-4.1.0.25 soundfile-0.10.3.post1 tensorboard-1.12.2 tensorflow-1.12.0 torch-1.1.0 torchvision-0.3.0 tqdm-4.45.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ey_bN4M6X_95",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 215
        },
        "outputId": "3d0a6395-1497-416c-f059-c57facc4b909"
      },
      "source": [
        "!wget \"https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth\" -O \"Wav2Lip/face_detection/detection/sfd/s3fd.pth\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-09-10 07:39:34--  https://www.adrianbulat.com/downloads/python-fan/s3fd-619a316812.pth\n",
            "Resolving www.adrianbulat.com (www.adrianbulat.com)... 45.136.29.207\n",
            "Connecting to www.adrianbulat.com (www.adrianbulat.com)|45.136.29.207|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 89843225 (86M) [application/octet-stream]\n",
            "Saving to: ‘Wav2Lip/face_detection/detection/sfd/s3fd.pth’\n",
            "\n",
            "Wav2Lip/face_detect 100%[===================>]  85.68M   144MB/s    in 0.6s    \n",
            "\n",
            "2020-09-10 07:39:34 (144 MB/s) - ‘Wav2Lip/face_detection/detection/sfd/s3fd.pth’ saved [89843225/89843225]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qdIQfY2Kswcb",
        "colab_type": "text"
      },
      "source": [
        "# Now lets try!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jR5utmDMcSZY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bfd3bc3e-de47-4ac7-c6b9-76d2ef2fee6c"
      },
      "source": [
        "!pwd\n",
        "!cp \"/content/gdrive/My Drive/Wav2lip/video.mp4\" \"/content/gdrive/My Drive/Wav2lip/audio.mp3\" sample_data/\n",
        "!ls sample_data/\n",
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"../sample_data/video.mp4\" --audio \"../sample_data/audio.mp3\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "anscombe.json  california_housing_test.csv   mnist_test.csv\t    README.md\n",
            "audio.mp3      california_housing_train.csv  mnist_train_small.csv  video.mp4\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
            "Using cuda for inference.\n",
            "Reading video frames...\n",
            "Number of frames available for inference: 690\n",
            "Extracting raw audio...\n",
            "ffmpeg version 3.4.8-0ubuntu0.2 Copyright (c) 2000-2020 the FFmpeg developers\n",
            "  built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)\n",
            "  configuration: --prefix=/usr --extra-version=0ubuntu0.2 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\n",
            "  libavutil      55. 78.100 / 55. 78.100\n",
            "  libavcodec     57.107.100 / 57.107.100\n",
            "  libavformat    57. 83.100 / 57. 83.100\n",
            "  libavdevice    57. 10.100 / 57. 10.100\n",
            "  libavfilter     6.107.100 /  6.107.100\n",
            "  libavresample   3.  7.  0 /  3.  7.  0\n",
            "  libswscale      4.  8.100 /  4.  8.100\n",
            "  libswresample   2.  9.100 /  2.  9.100\n",
            "  libpostproc    54.  7.100 / 54.  7.100\n",
            "Input #0, mp3, from '../sample_data/audio.mp3':\n",
            "  Metadata:\n",
            "    title           : 20200910 2966191144017524485\n",
            "    comment         : X2Convert.com, Best converter for all file type: Audio, Video...\n",
            "    encoder         : Lavf58.20.100\n",
            "  Duration: 00:00:23.07, start: 0.025057, bitrate: 122 kb/s\n",
            "    Chapter #0:0: start 0.000000, end 30.300000\n",
            "    Metadata:\n",
            "      title           : chp0\n",
            "    Stream #0:0: Audio: mp3, 44100 Hz, stereo, s16p, 122 kb/s\n",
            "    Metadata:\n",
            "      encoder         : LAME3.100\n",
            "Stream mapping:\n",
            "  Stream #0:0 -> #0:0 (mp3 (native) -> pcm_s16le (native))\n",
            "Press [q] to stop, [?] for help\n",
            "Output #0, wav, to 'temp/temp.wav':\n",
            "  Metadata:\n",
            "    INAM            : 20200910 2966191144017524485\n",
            "    ICMT            : X2Convert.com, Best converter for all file type: Audio, Video...\n",
            "    ISFT            : Lavf57.83.100\n",
            "    Chapter #0:0: start 0.000000, end 30.275000\n",
            "    Metadata:\n",
            "      title           : chp0\n",
            "    Stream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 44100 Hz, stereo, s16, 1411 kb/s\n",
            "    Metadata:\n",
            "      encoder         : Lavc57.107.100 pcm_s16le\n",
            "size=    3969kB time=00:00:23.04 bitrate=1411.3kbits/s speed= 336x    \n",
            "video:0kB audio:3969kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.004675%\n",
            "(80, 1844)\n",
            "Length of mel chunks: 686\n",
            "  0% 0/6 [00:00<?, ?it/s]\n",
            "  0% 0/43 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d7zgfrQqbKom",
        "colab_type": "text"
      },
      "source": [
        "## **Variations to try**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0f9A9VDVbZAG",
        "colab_type": "text"
      },
      "source": [
        "1.   Use more padding to include the chin region"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45XW4SZAzIz5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"../sample_data/input_vid.mp4\" --audio \"../sample_data/input_audio.wav\" --pads 0 20 0 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uo-WnsxfbwTG",
        "colab_type": "text"
      },
      "source": [
        "2.   Use resize_factor to reduce the video resolution, as there is a change you might get better results for lower resolution videos. Why? Because the model was trained on low resolution faces."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xw0xFtZ2bsx8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cd Wav2Lip && python inference.py --checkpoint_path checkpoints/wav2lip_gan.pth --face \"../sample_data/input_vid.mp4\" --audio \"../sample_data/input_audio.wav\" --resize_factor 2"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}