User_based_CF_colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.6"
    },
    "colab": {
      "name": "User_based_CF_colab.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arsh1207/BigDataProject/blob/master/User_based_CF_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xowopr87Tg7p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pyspark.rdd import RDD\n",
        "from pyspark.sql import Row\n",
        "from pyspark.sql import DataFrame\n",
        "from pyspark.sql import SparkSession\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "#import matplotlib.pyplot as plt\n",
        "import dask.array as da\n",
        "from numpy import cov\n",
        "from numpy import std\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "import random\n",
        "import math"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E_8sqoQcUN-4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://downloads.apache.org/spark/spark-3.0.0-preview2/spark-3.0.0-preview2-bin-hadoop2.7.tgz\n",
        "!tar -xvf spark-3.0.0-preview2-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IvtYUkf-UWB0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.0.0-preview2-bin-hadoop2.7\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fgyVaPM2UjsD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import findspark\n",
        "findspark.init()\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xGrUdYsMTg7x",
        "colab_type": "text"
      },
      "source": [
        "# User User based Collaborative filtering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0odGD6UmTg7z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def init_spark():\n",
        "    spark = SparkSession \\\n",
        "        .builder \\\n",
        "        .config(\"spark.driver.memory\", \"15g\") \\\n",
        "        .appName(\"Python Spark SQL basic example\") \\\n",
        "        .config(\"spark.some.config.option\", \"some-value\") \\\n",
        "        .getOrCreate()\n",
        "    return spark"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UeVsJTHvTg76",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "spark = init_spark()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4GCd-Vl2Yovl",
        "colab_type": "code",
        "outputId": "b4a06eec-1ed2-40ab-c267-257718b0d7ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rB7F6PbGZEUM",
        "colab_type": "code",
        "outputId": "aac05bcb-7241-4835-ad8f-7bb8f7f5510b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "!ls \"/content/drive/My Drive/Colab Notebooks\""
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " anime.csv\t\t\t\t\t   rating.csv\n",
            " Bike_Sharing.ipynb\t\t\t\t   test_data_animes.csv\n",
            " Classifier_Interpretability_Project.ipynb\t   test_data_user.csv\n",
            " Content_based_recommendation_system_Colab.ipynb   testfile2\n",
            "'Copy of Part1_TensorFlow.ipynb'\t\t   training_data.csv\n",
            "'Copy of Part2_Music_Generation.ipynb'\t\t   Untitled0.ipynb\n",
            " predicted\t\t\t\t\t   Untitled1.ipynb\n",
            " predicted1\t\t\t\t\t   User_based_CF_colab.ipynb\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VDVL6oJmTg8D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Reading Data from anime.csv\n",
        "df2 = pd.read_csv('/content/drive/My Drive/Colab Notebooks/anime.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_pgF4UumTg8J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Redaing from rating.csv and transforming data\n",
        "df = spark.read.text(\"/content/drive/My Drive/Colab Notebooks/rating.csv\").rdd\n",
        "data = df.map(lambda x: x[0].split(','))\n",
        "data2 = data.groupBy(lambda x : x[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OcHDxQ8KTg8P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data3 = data2.collect()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zhS_4tl5Tg8V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Preparing data : a dictionary utility = {users: {anime : ratings}}\n",
        "utility = {}\n",
        "all_animes = []\n",
        "all_users = []\n",
        "for user in range(len(data3)):\n",
        "    if data3[user][0] == 'user_id':\n",
        "        continue\n",
        "    utility[int(data3[user][0])] = {}\n",
        "    #if int(data3[user][0]) not in all_users:\n",
        "    #    all_users.append(int(i[0]))\n",
        "    for i in data3[user][1]:        \n",
        "        if int(i[1]) not in all_animes:\n",
        "            all_animes.append(int(i[1]))\n",
        "        if int(i[2]) == -1:\n",
        "            continue            \n",
        "        utility[int(i[0])][int(i[1])] = int(i[2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Ij9z_PyTg8w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_users = list(utility.keys())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7yJYzddTg8_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sim2(matrix, user1, user2):\n",
        "    #This method returns the similarity between two users\n",
        "    #It uses Pearson Correlation Coefficient for similarity calculation\n",
        "    matched = []\n",
        "    for anime in matrix[user1].keys():\n",
        "        if anime in matrix[user2].keys():\n",
        "            matched.append(anime) \n",
        "    x = []\n",
        "    y = []    \n",
        "    for anime in matched:\n",
        "        x.append(matrix[user1][anime])\n",
        "        y.append(matrix[user2][anime])        \n",
        "\n",
        "    f = (lambda i, j: i - j)\n",
        "    x = f(x, np.mean(x))\n",
        "    y = f(y, np.mean(y))\n",
        "    # Pearson Correlation Coefficient formula below\n",
        "    numerator = (lambda x, y : x*y)\n",
        "    denominator = (lambda x: x*x)\n",
        "    d = math.sqrt(sum(denominator(x))) * math.sqrt(sum(denominator(y)))\n",
        "    if d == 0:\n",
        "        return 0\n",
        "    similarity = sum(numerator(x, y))/ d\n",
        "    return similarity"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bc7LAzFyTg9C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def most_similar_user(matrix, user, n=500):\n",
        "    # This method returna a list of n most similar users\n",
        "    similarity = {}\n",
        "    for p in matrix.keys():\n",
        "        similarity[p] = sim2(matrix, user, p)\n",
        "    sorted_list = sorted(similarity.items(), key = lambda kv:(kv[1], kv[0]))\n",
        "    return sorted_list[-n:]     "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4EQesQ0Tg9E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def recommend(matrix, user):\n",
        "    # this method returns the predictions wrt most similar users\n",
        "    similar_users = most_similar_user(matrix, user)\n",
        "    predictions = {}\n",
        "    for anime in all_animes:\n",
        "        num = 0\n",
        "        den = 0\n",
        "        for user_ in similar_users:\n",
        "            if anime in matrix[user_[0]].keys():\n",
        "                num += matrix[user_[0]][anime] * user_[1]\n",
        "                den += user_[1]\n",
        "        if den == 0:\n",
        "            continue\n",
        "        #print(matrix[user[0]])\n",
        "        #if anime in matrix[user[0]].keys():\n",
        "        predictions[anime] = float(num/den)\n",
        "    return predictions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PjYdaa1-Tg9I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Creating Test data of randomly selecting 1000 users\n",
        "test= int(0.20 * len(all_users))\n",
        "test_data_user = []\n",
        "for _ in range(1000):\n",
        "    random_user = random.randint(1, len(all_users))\n",
        "    if all_users[random_user] not in test_data_user:\n",
        "        test_data_user.append(all_users[random_user])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OYJjDrMBXYnN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# loading the same test data\n",
        "test_data_users = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/test_data_user.csv\")\n",
        "test_data_animes = pd.read_csv(\"/content/drive/My Drive/Colab Notebooks/test_data_animes.csv\")\n",
        "test_data_user = test_data_users[\"user_id\"].to_list()\n",
        "test_data_animes = test_data_animes['test_animes'].to_list()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f-2yS7CshNSo",
        "colab_type": "code",
        "outputId": "3b3a7b93-93c9-4c63-8b74-4fd22fe369e0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(len(test_data_animes))"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3304\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kxbgWhrZTg9K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Creating test data of ramdomly selecting 30% of the total animes list\n",
        "test_animes = int(0.2 * len(all_animes))\n",
        "test_data_animes = []\n",
        "for _ in range(test_animes):\n",
        "    random_anime = random.randint(1, len(all_animes))\n",
        "    if all_animes[random_anime] not in test_data_animes:\n",
        "        test_data_animes.append(all_users[random_anime])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dmpet1-ZTg9N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# making a deep copy of the data \n",
        "import copy\n",
        "training_data = copy.deepcopy(utility)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hbfp-McwTg9P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Creating training data by deleting the test data from the original data\n",
        "for user in test_data_user:\n",
        "    for anime in test_data_animes:\n",
        "        if anime in training_data[user].keys():        \n",
        "            del training_data[user][anime]                        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rgu3CMbTh1Hr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Saving the test data of 1000 users\n",
        "#list_to_df = pd.DataFrame(test_data_user, columns =['user_id'])\n",
        "#list_to_df.to_csv (r'/content/drive/My Drive/Colab Notebooks/test_data_user.csv', index = False, header=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22nE7fb8Tg9S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def start():\n",
        "    predicted = {}\n",
        "    counter = 0\n",
        "    for user in test_data_user:\n",
        "        predicted[user] = recommend(training_data, user)\n",
        "        counter += 1\n",
        "        if counter%50 ==0:\n",
        "          print(counter)\n",
        "    return predicted"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JSWuEVQATg9U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 403
        },
        "outputId": "1efdc5ac-18a4-4301-aba8-37589c461d06"
      },
      "source": [
        "\n",
        "predicted1 = start()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
            "  out=out, **kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "50\n",
            "100\n",
            "150\n",
            "200\n",
            "250\n",
            "300\n",
            "350\n",
            "400\n",
            "450\n",
            "500\n",
            "550\n",
            "600\n",
            "650\n",
            "700\n",
            "750\n",
            "800\n",
            "850\n",
            "900\n",
            "950\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_MNfwzMebmn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def RMSE(actual, predicted):\n",
        "    error = 0\n",
        "    n = 0\n",
        "    for user in test_data_user:\n",
        "        for anime in test_data_animes:\n",
        "            if anime in predicted[user].keys() and anime in actual[user].keys():\n",
        "                error += (actual[user][anime] - predicted[user][anime])**2\n",
        "                n += 1\n",
        "    return math.sqrt(error/n)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "48bIxxqpTg9Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "17650b1d-875a-4a24-b0ce-c76e34ff2074"
      },
      "source": [
        "RMSE(utility, predicted1)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.7525965512669202"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sm9othsPTg9a",
        "colab_type": "text"
      },
      "source": [
        "## Applying the common practice "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ia7tKJjvejXr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "avg_rating_of_user = {}\n",
        "global_avg = 0\n",
        "counter = 0\n",
        "for user in all_users:\n",
        "    count = 0\n",
        "    summ = 0\n",
        "    for anime in utility[user].keys():\n",
        "        summ += utility[user][anime]\n",
        "        count += 1\n",
        "        global_avg += utility[user][anime]\n",
        "        counter += 1\n",
        "    if count != 0:\n",
        "        avg_rating_of_user[user] = summ/count\n",
        "    else:\n",
        "        avg_rating_of_user[user] = 0\n",
        "global_avg = global_avg/counter"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FllfdC11xsbi",
        "colab_type": "code",
        "outputId": "6962ded4-0760-40a5-f945-a80fe6f09f9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "global_avg"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7.808496893124035"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LPkEIt_ITg9a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "avg_rating_of_animes = {}\n",
        "for anime in all_animes:\n",
        "    count = 0\n",
        "    summ = 0\n",
        "    for user in all_users:\n",
        "        if anime in utility[user].keys():\n",
        "            summ += utility[user][anime]\n",
        "            count += 1\n",
        "    if count != 0:\n",
        "        avg_rating_of_animes[anime] = summ/count\n",
        "    else:\n",
        "        avg_rating_of_animes[anime] = 0\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bG5HBMNXTg9c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def recommend_with_regularization(matrix, user):\n",
        "    # this method returns the predictions wrt most similar users\n",
        "    similar_users = most_similar_user(matrix, user)\n",
        "    predictions = {}\n",
        "    for anime in all_animes:\n",
        "        num = 0\n",
        "        den = 0\n",
        "        for user_ in similar_users:\n",
        "            if anime in matrix[user_[0]].keys():\n",
        "                b_xj = global_avg + (avg_rating_of_user[user_[0]] - global_avg) + (avg_rating_of_animes[anime]- global_avg)\n",
        "                num += (matrix[user_[0]][anime]- b_xj) * user_[1]\n",
        "                den += user_[1]\n",
        "        if den == 0:\n",
        "            continue\n",
        "        # if anime in matrix[user[0]].keys():\n",
        "        b_xi = global_avg + (avg_rating_of_user[user] - global_avg) + (avg_rating_of_animes[anime] - global_avg)\n",
        "        if b_xi + float(num/den) > 10:\n",
        "            predictions[anime] = 10\n",
        "        else:\n",
        "            predictions[anime] = b_xi + float(num/den)\n",
        "    return predictions"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STh55fT6i1SC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def start2():\n",
        "    predicted = {}\n",
        "    counter = 0\n",
        "    for user in test_data_user:\n",
        "        predicted[user] = recommend_with_regularization(training_data, user)\n",
        "        counter += 1\n",
        "        if counter%50 ==0:\n",
        "            print(counter)\n",
        "    return predicted"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "STNGR_TEkout",
        "colab_type": "code",
        "outputId": "6f8119d2-fee5-4202-f9a3-fbbe9bde7ede",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 403
        }
      },
      "source": [
        "predicted = start2()"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
            "  out=out, **kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "50\n",
            "100\n",
            "150\n",
            "200\n",
            "250\n",
            "300\n",
            "350\n",
            "400\n",
            "450\n",
            "500\n",
            "550\n",
            "600\n",
            "650\n",
            "700\n",
            "750\n",
            "800\n",
            "850\n",
            "900\n",
            "950\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qlJ70y3jmJjI",
        "colab_type": "code",
        "outputId": "886efc26-73d3-447d-94ec-702eab3bb76d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "RMSE(utility, predicted)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.4800267481453462"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PlyxJ5xV9I_G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Saving the predictions\n",
        "import pickle\n",
        "file_Name = \"/content/drive/My Drive/Colab Notebooks/predicted\"\n",
        "\n",
        "with open(file_Name, 'wb') as handle:\n",
        "    pickle.dump(predicted, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-r6eE_l4AD6A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Loading the saved predicted files\n",
        "file_Name = \"/content/drive/My Drive/Colab Notebooks/testfile2\"\n",
        "with open(file_Name, 'rb') as handle:\n",
        "    predicted3 = pickle.load(handle)\n",
        "\n",
        "#predicted == b\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqC2rGcBTg9p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def recommend_animes(for_user, n = 10):\n",
        "    predictions = recommend(training_data, for_user)\n",
        "    sorted_list = sorted(predictions.items(), key = lambda kv:(kv[1], kv[0]))\n",
        "    for anime in sorted_list[-n:]:\n",
        "        values = df2[['anime_id','name', 'genre', 'type']].loc[df2['anime_id']==anime[0]]\n",
        "        print(str(values['anime_id']) + \",\" + str(values['name']) + \",\" + str(values['genre']) + \",\" + str(values['type']) + \"\\n\")\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YWuNdb5ATg9s",
        "colab_type": "code",
        "outputId": "856d8084-6553-49a0-ddea-1512c6c3173f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "recommend_animes(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
            "  out=out, **kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "113    30503\n",
            "Name: anime_id, dtype: int64,113    Noragami Aragoto\n",
            "Name: name, dtype: object,113    Action, Adventure, Shounen, Supernatural\n",
            "Name: genre, dtype: object,113    TV\n",
            "Name: type, dtype: object\n",
            "\n",
            "1817    30991\n",
            "Name: anime_id, dtype: int64,1817    Himouto! Umaru-chan: Umaru-chan Mou Ikkai!\n",
            "Name: name, dtype: object,1817    Comedy, School, Seinen, Slice of Life\n",
            "Name: genre, dtype: object,1817    OVA\n",
            "Name: type, dtype: object\n",
            "\n",
            "4954    31121\n",
            "Name: anime_id, dtype: int64,4954    Monster Musume no Iru Nichijou: Hobo Mainichi ...\n",
            "Name: name, dtype: object,4954    Comedy, Slice of Life\n",
            "Name: genre, dtype: object,4954    ONA\n",
            "Name: type, dtype: object\n",
            "\n",
            "369    31173\n",
            "Name: anime_id, dtype: int64,369    Akagami no Shirayuki-hime 2nd Season\n",
            "Name: name, dtype: object,369    Drama, Fantasy, Romance, Shoujo\n",
            "Name: genre, dtype: object,369    TV\n",
            "Name: type, dtype: object\n",
            "\n",
            "3375    31234\n",
            "Name: anime_id, dtype: int64,3375    Himouto! Umaru-chanS\n",
            "Name: name, dtype: object,3375    Comedy, Seinen, Slice of Life\n",
            "Name: genre, dtype: object,3375    Special\n",
            "Name: type, dtype: object\n",
            "\n",
            "3306    31442\n",
            "Name: anime_id, dtype: int64,3306    Musaigen no Phantom World\n",
            "Name: name, dtype: object,3306    Action, Comedy, Fantasy, Slice of Life, Supern...\n",
            "Name: genre, dtype: object,3306    TV\n",
            "Name: type, dtype: object\n",
            "\n",
            "638    31711\n",
            "Name: anime_id, dtype: int64,638    To LOVE-Ru Darkness 2nd Specials\n",
            "Name: name, dtype: object,638    Comedy, Ecchi, Harem, Romance, School, Sci-Fi,...\n",
            "Name: genre, dtype: object,638    Special\n",
            "Name: type, dtype: object\n",
            "\n",
            "361    32122\n",
            "Name: anime_id, dtype: int64,361    Gintama: Jump Festa 2015 Special\n",
            "Name: name, dtype: object,361    Action, Comedy, Historical, Parody, Samurai, S...\n",
            "Name: genre, dtype: object,361    Special\n",
            "Name: type, dtype: object\n",
            "\n",
            "1822    32574\n",
            "Name: anime_id, dtype: int64,1822    Kekkai Sensen: Ousama no Restaurant no Ousama\n",
            "Name: name, dtype: object,1822    Action, Fantasy, Shounen, Super Power, Superna...\n",
            "Name: genre, dtype: object,1822    OVA\n",
            "Name: type, dtype: object\n",
            "\n",
            "798    33082\n",
            "Name: anime_id, dtype: int64,798    Kingsglaive: Final Fantasy XV\n",
            "Name: name, dtype: object,798    Action\n",
            "Name: genre, dtype: object,798    Movie\n",
            "Name: type, dtype: object\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DkSb5xVm0xgL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def recommend_animes_regularized(for_user, n = 10):\n",
        "    predictions = recommend_with_regularization(training_data, for_user)\n",
        "    sorted_list = sorted(predictions.items(), key = lambda kv:(kv[1], kv[0]))\n",
        "    for anime in sorted_list[-n:]:\n",
        "        values = df2[['anime_id','name', 'genre', 'type']].loc[df2['anime_id']==anime[0]]\n",
        "        print(str(values['anime_id']) + \",\" + str(values['name']) + \",\" + str(values['genre']) + \",\" + str(values['type']) + \"\\n\")\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "De6jH1yZ0TiL",
        "colab_type": "code",
        "outputId": "088178bb-6574-409f-be62-d8feb0017c2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "recommend_animes_regularized(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
            "  out=out, **kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1156    1488\n",
            "Name: anime_id, dtype: int64,1156    Area 88\n",
            "Name: name, dtype: object,1156    Action, Adventure, Drama, Military, Romance\n",
            "Name: genre, dtype: object,1156    OVA\n",
            "Name: type, dtype: object\n",
            "\n",
            "1546    1577\n",
            "Name: anime_id, dtype: int64,1546    Taiho Shichau zo\n",
            "Name: name, dtype: object,1546    Action, Comedy, Police, Seinen\n",
            "Name: genre, dtype: object,1546    OVA\n",
            "Name: type, dtype: object\n",
            "\n",
            "2172    1901\n",
            "Name: anime_id, dtype: int64,2172    11-nin Iru!\n",
            "Name: name, dtype: object,2172    Action, Adventure, Drama, Mystery, Romance, Sc...\n",
            "Name: genre, dtype: object,2172    Movie\n",
            "Name: type, dtype: object\n",
            "\n",
            "3259    1951\n",
            "Name: anime_id, dtype: int64,3259    Manie-Manie: Meikyuu Monogatari\n",
            "Name: name, dtype: object,3259    Adventure, Fantasy, Horror, Sci-Fi, Supernatural\n",
            "Name: genre, dtype: object,3259    Movie\n",
            "Name: type, dtype: object\n",
            "\n",
            "1835    2182\n",
            "Name: anime_id, dtype: int64,1835    Robot Carnival\n",
            "Name: name, dtype: object,1835    Fantasy, Sci-Fi\n",
            "Name: genre, dtype: object,1835    OVA\n",
            "Name: type, dtype: object\n",
            "\n",
            "4748    2699\n",
            "Name: anime_id, dtype: int64,4748    Uchuu Kaizoku Mito no Daibouken\n",
            "Name: name, dtype: object,4748    Action, Comedy, Sci-Fi\n",
            "Name: genre, dtype: object,4748    TV\n",
            "Name: type, dtype: object\n",
            "\n",
            "5302    4475\n",
            "Name: anime_id, dtype: int64,5302    Toshi wo Totta Wani\n",
            "Name: name, dtype: object,5302    Drama\n",
            "Name: genre, dtype: object,5302    Movie\n",
            "Name: type, dtype: object\n",
            "\n",
            "2441    6361\n",
            "Name: anime_id, dtype: int64,2441    ef: A Tale of Memories. - Prologue\n",
            "Name: name, dtype: object,2441    Drama, Music\n",
            "Name: genre, dtype: object,2441    Special\n",
            "Name: type, dtype: object\n",
            "\n",
            "962    10278\n",
            "Name: anime_id, dtype: int64,962    The iDOLM@STER\n",
            "Name: name, dtype: object,962    Comedy, Drama, Music\n",
            "Name: genre, dtype: object,962    TV\n",
            "Name: type, dtype: object\n",
            "\n",
            "616    20159\n",
            "Name: anime_id, dtype: int64,616    Pokemon: The Origin\n",
            "Name: name, dtype: object,616    Action, Adventure, Comedy, Fantasy, Kids\n",
            "Name: genre, dtype: object,616    Special\n",
            "Name: type, dtype: object\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}