Pytorch_(RNN).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mamintoosi/Text-Mining/blob/master/Pytorch_(RNN).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mVhK2zxVPtU0"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = ['hey how are you','good i am fine','have a nice day']\n",
        "\n",
        "# Join all the sentences together and extract the unique characters from the combined sentences\n",
        "chars = set(''.join(text))\n",
        "\n",
        "# Creating a dictionary that maps integers to the characters\n",
        "int2char = dict(enumerate(chars))\n",
        "\n",
        "# Creating another dictionary that maps characters to integers\n",
        "char2int = {char: ind for ind, char in int2char.items()}"
      ],
      "metadata": {
        "id": "mG01cSBbPwC9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (chars)\n",
        "print (int2char)\n",
        "print (char2int)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pbvh6zVoP1iw",
        "outputId": "442f2fe2-9329-4f68-e044-747361c2f545"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'f', 'a', 'g', 'i', 'u', 'm', 'v', 'c', 'h', 'o', 'y', 'r', ' ', 'w', 'n', 'e', 'd'}\n",
            "{0: 'f', 1: 'a', 2: 'g', 3: 'i', 4: 'u', 5: 'm', 6: 'v', 7: 'c', 8: 'h', 9: 'o', 10: 'y', 11: 'r', 12: ' ', 13: 'w', 14: 'n', 15: 'e', 16: 'd'}\n",
            "{'f': 0, 'a': 1, 'g': 2, 'i': 3, 'u': 4, 'm': 5, 'v': 6, 'c': 7, 'h': 8, 'o': 9, 'y': 10, 'r': 11, ' ': 12, 'w': 13, 'n': 14, 'e': 15, 'd': 16}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Finding the length of the longest string in our data\n",
        "maxlen = len(max(text, key=len))\n",
        "\n",
        "# Padding\n",
        "\n",
        "# A simple loop that loops through the list of sentences and adds a ' ' whitespace until the length of\n",
        "# the sentence matches the length of the longest sentence\n",
        "for i in range(len(text)):\n",
        "  while len(text[i])<maxlen:\n",
        "      text[i] += ' '"
      ],
      "metadata": {
        "id": "iAwhkvByQCfK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating lists that will hold our input and target sequences\n",
        "input_seq = []\n",
        "target_seq = []\n",
        "\n",
        "for i in range(len(text)):\n",
        "    # Remove last character for input sequence\n",
        "  input_seq.append(text[i][:-1])\n",
        "\n",
        "    # Remove first character for target sequence\n",
        "  target_seq.append(text[i][1:])\n",
        "  print(\"Input Sequence: {}\\nTarget Sequence: {}\".format(input_seq[i], target_seq[i]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OCnXHXblQIPj",
        "outputId": "58e2e2c3-8547-4129-a079-6f41609c0efa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input Sequence: hey how are yo\n",
            "Target Sequence: ey how are you\n",
            "Input Sequence: good i am fine\n",
            "Target Sequence: ood i am fine \n",
            "Input Sequence: have a nice da\n",
            "Target Sequence: ave a nice day\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(text)):\n",
        "    input_seq[i] = [char2int[character] for character in input_seq[i]]\n",
        "    target_seq[i] = [char2int[character] for character in target_seq[i]]\n"
      ],
      "metadata": {
        "id": "_rXQqU8kQMHq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dict_size = len(char2int)\n",
        "seq_len = maxlen - 1\n",
        "batch_size = len(text)\n",
        "\n",
        "def one_hot_encode(sequence, dict_size, seq_len, batch_size):\n",
        "    # Creating a multi-dimensional array of zeros with the desired output shape\n",
        "    features = np.zeros((batch_size, seq_len, dict_size), dtype=np.float32)\n",
        "\n",
        "    # Replacing the 0 at the relevant character index with a 1 to represent that character\n",
        "    for i in range(batch_size):\n",
        "        for u in range(seq_len):\n",
        "            features[i, u, sequence[i][u]] = 1\n",
        "    return features"
      ],
      "metadata": {
        "id": "f9w2qULFQQzh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Input shape --> (Batch Size, Sequence Length, One-Hot Encoding Size)\n",
        "input_seq = one_hot_encode(input_seq, dict_size, seq_len, batch_size)"
      ],
      "metadata": {
        "id": "SytU0le4QTqn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_seq = torch.from_numpy(input_seq)\n",
        "target_seq = torch.Tensor(target_seq)"
      ],
      "metadata": {
        "id": "p6Bj7THRQWQa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.cuda.is_available() checks and returns a Boolean True if a GPU is available, else it'll return False\n",
        "is_cuda = torch.cuda.is_available()\n",
        "\n",
        "# If we have a GPU available, we'll set our device to GPU. We'll use this device variable later in our code.\n",
        "if is_cuda:\n",
        "    device = torch.device(\"cuda\")\n",
        "    print(\"GPU is available\")\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    print(\"GPU not available, CPU used\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P2Y45bAxQZAL",
        "outputId": "cfb9e0cb-f061-44e3-af08-63f9e43411ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPU not available, CPU used\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Model(nn.Module):\n",
        "    def __init__(self, input_size, output_size, hidden_dim, n_layers):\n",
        "        super(Model, self).__init__()\n",
        "\n",
        "        # Defining some parameters\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.n_layers = n_layers\n",
        "\n",
        "        #Defining the layers\n",
        "        # RNN Layer\n",
        "        self.rnn = nn.RNN(input_size, hidden_dim, n_layers, batch_first=True)\n",
        "        # Fully connected layer\n",
        "        self.fc = nn.Linear(hidden_dim, output_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "\n",
        "        batch_size = x.size(0)\n",
        "\n",
        "        # Initializing hidden state for first input using method defined below\n",
        "        hidden = self.init_hidden(batch_size)\n",
        "\n",
        "        # Passing in the input and hidden state into the model and obtaining outputs\n",
        "        out, hidden = self.rnn(x, hidden)\n",
        "\n",
        "        # Reshaping the outputs such that it can be fit into the fully connected layer\n",
        "        out = out.contiguous().view(-1, self.hidden_dim)\n",
        "        out = self.fc(out)\n",
        "\n",
        "        return out, hidden\n",
        "\n",
        "    def init_hidden(self, batch_size):\n",
        "        # This method generates the first hidden state of zeros which we'll use in the forward pass\n",
        "        # We'll send the tensor holding the hidden state to the device we specified earlier as well\n",
        "        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim)\n",
        "        return hidden"
      ],
      "metadata": {
        "id": "LX4sRAabQb65"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate the model with hyperparameters\n",
        "model = Model(input_size=dict_size, output_size=dict_size, hidden_dim=12, n_layers=1)\n",
        "# We'll also set the model to the device that we defined earlier (default is CPU)\n",
        "model.to(device)\n",
        "\n",
        "# Define hyperparameters\n",
        "n_epochs = 100\n",
        "lr=0.01\n",
        "\n",
        "# Define Loss, Optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=lr)"
      ],
      "metadata": {
        "id": "9t4n6id_QgFg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Training Run\n",
        "for epoch in range(1, n_epochs + 1):\n",
        "    optimizer.zero_grad() # Clears existing gradients from previous epoch\n",
        "    input_seq.to(device)\n",
        "    output, hidden = model(input_seq)\n",
        "    loss = criterion(output, target_seq.view(-1).long())\n",
        "    loss.backward() # Does backpropagation and calculates gradients\n",
        "    optimizer.step() # Updates the weights accordingly\n",
        "\n",
        "    if epoch%10 == 0:\n",
        "        print('Epoch: {}/{}.............'.format(epoch, n_epochs), end=' ')\n",
        "        print(\"Loss: {:.4f}\".format(loss.item()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FBSABjOfQjO0",
        "outputId": "0f42c799-0588-4ee4-f8af-3f1ea2526b8a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 10/100............. Loss: 2.3797\n",
            "Epoch: 20/100............. Loss: 2.0657\n",
            "Epoch: 30/100............. Loss: 1.6512\n",
            "Epoch: 40/100............. Loss: 1.2376\n",
            "Epoch: 50/100............. Loss: 0.8874\n",
            "Epoch: 60/100............. Loss: 0.6297\n",
            "Epoch: 70/100............. Loss: 0.4469\n",
            "Epoch: 80/100............. Loss: 0.3251\n",
            "Epoch: 90/100............. Loss: 0.2456\n",
            "Epoch: 100/100............. Loss: 0.1911\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# This function takes in the model and character as arguments and returns the next character prediction and hidden state\n",
        "def predict(model, character):\n",
        "    # One-hot encoding our input to fit into the model\n",
        "    character = np.array([[char2int[c] for c in character]])\n",
        "    character = one_hot_encode(character, dict_size, character.shape[1], 1)\n",
        "    character = torch.from_numpy(character)\n",
        "    character.to(device)\n",
        "\n",
        "    out, hidden = model(character)\n",
        "\n",
        "    prob = nn.functional.softmax(out[-1], dim=0).data\n",
        "    # Taking the class with the highest probability score from the output\n",
        "    char_ind = torch.max(prob, dim=0)[1].item()\n",
        "\n",
        "    return int2char[char_ind], hidden"
      ],
      "metadata": {
        "id": "GkbVB0jIQlsZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This function takes the desired output length and input characters as arguments, returning the produced sentence\n",
        "def sample(model, out_len, start='hey'):\n",
        "    model.eval() # eval mode\n",
        "    start = start.lower()\n",
        "    # First off, run through the starting characters\n",
        "    chars = [ch for ch in start]\n",
        "    size = out_len - len(chars)\n",
        "    # Now pass in the previous characters and get a new one\n",
        "    for ii in range(size):\n",
        "        char, h = predict(model, chars)\n",
        "        chars.append(char)\n",
        "\n",
        "    return ''.join(chars)"
      ],
      "metadata": {
        "id": "pS-ENgjHQpt0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sample(model, 15, 'good')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "O2Y7nKK0QtN6",
        "outputId": "fa42734a-fdc6-44ba-f5a4-2189638d7507"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'good i am fine '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample(model, 15, 'hey')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "g1-HoTf2QvSK",
        "outputId": "d3e87323-0918-4925-eb8c-3d06b9fbcdb6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'hey how are you'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sample(model, 15, 'have')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "h_IYGog6Q5ib",
        "outputId": "68332843-fe01-419d-c1cf-23eea0fcd2b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'have a nice day'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_9cJXr-UQ-Vw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "\n",
        "---\n",
        "\n",
        "---\n",
        "\n",
        "---\n",
        "\n",
        "---"
      ],
      "metadata": {
        "id": "4YN7D7bF0eUA"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Conference\n",
        "\n",
        "A novel recurrent neural network based of NCP function for solving convex quadratic programming problems\n",
        "\n",
        "* Sohrab Effati\n",
        "\n",
        "* The 2nd international Conference on Control and Optimization with Industrial Applications (2008-06-02)\n",
        "\n",
        "A RECURRENT NEURAL NETWORK FOR SOLVING NONCONVEX NONLINEAR OPTIMIZATION PROBLEM\n",
        "\n",
        "* Mohammad Moghaddas - Sohrab Effati\n",
        "\n",
        "* The 44 th Annual Iranian Mathematics Conference Ferdowsi University of Mashhad, (27-30 August 2013)\n",
        "\n",
        "An projection recurrent neural network model to solve the fuzzy shortest path problem\n",
        "\n",
        "* Mohammad Eshaghnezhad - Freydoon Rahbarnia - Sohrab Effati\n",
        "\n",
        "* هشتمین سمینار آمار و احتمال فازی (2018-05-09)\n",
        "\n",
        "A Projection RNN For Non-Convex Optimization Problems\n",
        "\n",
        "* Amin Mansoori - Sohrab Effati - Mohammad Eshaghnezhad\n",
        "\n",
        "* The Third National Seminar on Control and Optimization (2019-11-13)\n",
        "\n",
        "---\n",
        "\n",
        "A recurrent neural network-based method for training probabilistic Support Vector Machine\n",
        "\n",
        "* Hadi Sadoghi Yazdi ، Sohrab Effati\n",
        "\n",
        "* International Journal of Signal and Imaging Systems Engineering-IJSISE (2009 November)\n",
        "\n",
        "A novel recurrent nonlinear neural network for solving quadratic programming problems\n",
        "\n",
        "* Sohrab Effati\n",
        "\n",
        "* Applied Mathematical Modelling (2011 January)\n",
        "\n",
        "A Novel Recurrent Neural Network for Solving Mlcps and its Application to Linear and Quadratic Programming\n",
        "\n",
        "* Sohrab Effati\n",
        "\n",
        "* Asia-Pacific Journal of Operational Research (2011 November)\n",
        "\n",
        "An efficient recurrent neural network model for solving fuzzy non-linear programming problems\n",
        "\n",
        "* Amin Mansoori ، Sohrab Effati ، Mohammad Eshaghnezhad\n",
        "\n",
        "* Applied Intelligence (2016 January)\n",
        "\n",
        "Recurrent Neural Network Model: A New Strategy to Solve Fuzzy Matrix Games\n",
        "\n",
        "* Amin Mansoori ، Mohammad Eshaghnezhad ، Sohrab Effati\n",
        "\n",
        "* IEEE Transactions on Neural Networks and Learning Systems (2018 December)\n",
        "\n",
        "Parametric NCP-Based Recurrent Neural Network Model: A New Strategy to Solve Fuzzy Nonconvex Optimization Problems\n",
        "\n",
        "* Amin Mansoori ، Sohrab Effati\n",
        "\n",
        "* IEEE Transactions on Systems, Man, and Cybernetics: Systems (2019 January)\n",
        "\n",
        "Projection Recurrent Neural Network Model: A New Strategy to Solve Maximum Flow Problem\n",
        "\n",
        "* Mohammad Eshaghnezhad ، Sohrab Effati ، Freydoon Rahbarnia\n",
        "\n",
        "* IEEE Transactions on Circuits and Systems Part II: Express Briefs (2020 January)\n",
        "\n",
        "A compact MLCP-based projection recurrent neural network model to solve shortest path problem\n",
        "\n",
        "* Mohammad Eshaghnezhad ، Sohrab Effati ، Amin Mansoori\n",
        "\n",
        "* Journal of Experimental and Theoretical Artificial Intelligence (2022 April)\n",
        "\n",
        "---\n",
        "\n",
        "**Abstract:**\n",
        "This paper aims to investigate the fuzzy constrained matrix game -MG- problems using the concepts of recurrent neural networks -RNNs-. To the best of our knowledge, this paper is the first in attempting to find a solution for fuzzy game problems using RNN models. For this purpose, a fuzzy game problem is reformulated into a weighting problem. Then, the Karush–Kuhn–Tucker -KKT- optimality conditions are provided for the weighting problem. The KKT conditions are used to propose the RNN model. Moreover, the Lyapunov stability and the global convergence of the RNN model are also confirmed. Finally, three illustrative examples are presented to demonstrate the effectiveness of this approach. The obtained results are compared with the results obtained by the previous approaches for solving fuzzy constrained MG.\n",
        "\n",
        "**چکیده:**\n",
        "هدف این مقاله بررسی مسائل بازی ماتریس محدود فازی با استفاده از مفاهیم شبکه های عصبی بازگشتی است. با بهترین دانش ما، این مقاله اولین مقاله ای است که در تلاش برای یافتن راه حلی برای مسائل بازی فازی با استفاده از مدل های شبکه عصبی بازگشتی است. برای این منظور، یک مسئله بازی فازی به یک مسئله وزنی مجدداً فرموله می شود. سپس شرایط بهینه کاروش-کوهن-تاکر برای مسئله وزن دهی ارائه می شود. شرایط کاروش-کوهن-تاکر برای پیشنهاد مدل شبکه عصبی بازگشتی استفاده می شود. علاوه بر این، پایداری لیاپانوف و همگرایی جهانی مدل شبکه عصبی بازگشتی نیز تایید شده است. در نهایت، سه مثال گویا برای نشان دادن اثربخشی این رویکرد ارائه شده است. نتایج به‌دست‌آمده با نتایج به‌دست‌آمده از روش‌های قبلی برای حل MG مقید فازی مقایسه می‌شود"
      ],
      "metadata": {
        "id": "qbKPzQSO05df"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "M-qtVGvx3moJ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}