ConfirmMaps.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confirm generic conceptual models using Social media/Academic data"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# coding: utf-8\n",
    "\n",
    "# Developed  by: Mannila Sandhu\n",
    "# Supervised by: Philippe Giabbanelli, Vijay Mago\n",
    "# Copyright (c) 2018,\n",
    "# (DACHB) Lab, FURMAN UNIVERSITY; DATALAB, LAKEHEAD UNIVERSITY\n",
    "# All rights reserved\n",
    "\n",
    "\"\"\"\n",
    "Author: Mannila Sandhu (msandhu3@lakeheadu.ca)\n",
    "\n",
    "To confirm the causal map using Twitter/Academic data\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Necessary import libraries\"\"\"\n",
    "\n",
    "import os\n",
    "import time\n",
    "import csv\n",
    "import json\n",
    "import re\n",
    "import pandas as pd\n",
    "from itertools import groupby\n",
    "import networkx as nx\n",
    "from networkx.algorithms import approximation as approx\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')\n",
    "from gensim import corpora, models\n",
    "import gensim\n",
    "from google.cloud import language \n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Global variables\"\"\"\n",
    "\n",
    "debug=False #use this code to debug as per requiremnet(if debug==True: print('DESIRED DEBUG VALUE'))\n",
    "n_topics = 50\n",
    "n_words = 50\n",
    "max_df = 9\n",
    "#input data files\n",
    "tweetDataset= './data/lemma_pdf_text.json'\n",
    "mapFile='./maps/maps5.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"To read tweets/sentences from JSON file\"\"\"\n",
    "\n",
    "tweets = []\n",
    "\n",
    "with open(tweetDataset, 'r', encoding='utf-8-sig') as filehandle:  \n",
    "    tweets = json.load(filehandle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Input graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Create a Directed usermap from a file\"\"\"\n",
    "\n",
    "usermap = nx.DiGraph()\n",
    "f1 = csv.reader(open(mapFile,\"r\"))\n",
    "for row in f1:\n",
    "    usermap.add_edge(row[0],row[1]) #, weight = row[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"To view or save the Directed usermap created\"\"\"\n",
    "# can change size of plot for bigger maps display\n",
    "plt.rcParams['figure.figsize'] = [10, 7] # [width, height] of plot\n",
    "# To visualize the graph\n",
    "nx.draw_kamada_kawai(usermap, with_labels=True, font_size=15, node_color='lightblue', node_size=1000)\n",
    "plt.savefig(\"./usermap.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](./img/pic1.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Derivationally related forms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"To fetch derivationally related forms of the nodes of usermap\"\"\"\n",
    "\n",
    "der_rel_form = {}\n",
    "for node in usermap.nodes():                                            #for each node of the concept map\n",
    "    der_rel_form_values = set()                                         #set of list of derivationally related forms\n",
    "    der_rel_form_values.add(node)            \n",
    "    phrases = word_tokenize(node)                                    #for node consisting of more than one word\n",
    "    for token in phrases:\n",
    "        der_rel_form_values.add(token)\n",
    "        for each_synsets in wn.synsets(token):                          #Look up a word using synsets()\n",
    "            for each_lemma in each_synsets.lemmas():                    #Each synset contains one or more lemmas, which represent a specific sense of a specific word\n",
    "                for each in each_lemma.derivationally_related_forms():  #derivationally_related_forms relation is defined by WordNet only over Lemmas\n",
    "                    der_rel_form_values.add(each.name().lower())\n",
    "    der_rel_form[node] = der_rel_form_values\n",
    "\n",
    "if debug==True:\n",
    "    with open('./der_rel_form.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in der_rel_form:\n",
    "            outfile.write(key+':'+str(der_rel_form[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](./img/pic2.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve non emtpy relevant tweets/sentences and find out the empty nodes removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"To retrieve relevant tweets/sentences using derivationally related forms of the nodes of usermap\"\"\"\n",
    "\n",
    "relevantTweets = {}\n",
    "for k,v in der_rel_form.items():\n",
    "    relevantTweets_values = set()                           #set of list of relevant tweets\n",
    "    for keywords in v:                                      #lookup for keywords in those lists inside the dictionary iteratively\n",
    "        for tweet in tweets:                                #lookup for single tweet inside the input 'tweets'\n",
    "            if (' ' + keywords + ' ') in (' ' + tweet + ' '):\n",
    "                relevantTweets_values.add(tweet)            #add relevant tweet according to the presence of keywords among them\n",
    "    relevantTweets[k] = relevantTweets_values\n",
    "\n",
    "nonEmtyRelevantTweets = dict((k, v) for k, v in relevantTweets.items() if v) #removes keys with empty key-value\n",
    "\n",
    "if debug==True:\n",
    "    with open('./nonEmtyRelevantTweets.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in nonEmtyRelevantTweets:\n",
    "            outfile.write(key+':'+str(nonEmtyRelevantTweets[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# To view the empty nodes removed, if any"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodesremoved = set()\n",
    "for k,v in relevantTweets.items():                          #to print the empty nodes\n",
    "    if not v:\n",
    "        nodesremoved.add(k)\n",
    "        \n",
    "print(\"nodes removed : \" + str(nodesremoved))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve total number of relevant tweets/sentences for given user data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"To retrieve number of relevant tweets/sentences\"\"\"\n",
    "\n",
    "unique = set()\n",
    "for v in nonEmtyRelevantTweets.values():\n",
    "    for i in v:\n",
    "        unique.add(i)\n",
    "        \n",
    "print(len(unique))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve number of relevant tweets corresponding to each node of user map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to retrieve the number of relevant tweets/sentences collected per node of map\"\"\"\n",
    "\n",
    "key_to_value_lengths = {k:[len(v)] for k, v in nonEmtyRelevantTweets.items()}\n",
    "\n",
    "if debug==True:\n",
    "    with open('./key_to_value_lengths.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in key_to_value_lengths:\n",
    "            outfile.write(key+':'+str(key_to_value_lengths[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](./img/pic3.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run LDA multicore model to fetch themes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to fetch relevant themes\"\"\" \n",
    "themes = {}\n",
    "for k,v in nonEmtyRelevantTweets.items():  #for each key,value of the hashmap 'nonEmtyRelevantTweets'\n",
    "    themes_values = set()                  #set of list of relevant themes\n",
    "    texts = [[word for word in document.split()] for document in v] #splits sentences into words\n",
    "    #traverses texts, assigning a unique integer id to each unique token while also collecting word counts and relevant statistics\n",
    "    dictionary = corpora.Dictionary(texts)\n",
    "    #convert dictionary into a bag-of-words\n",
    "    corpus = [dictionary.doc2bow(text) for text in texts]  #corpus created is a list of vectors equal to the number of documents. \n",
    "    #In each document vector is a series of tuples(termid,term freq)\n",
    "    lda = gensim.models.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=n_topics,\n",
    "                                     chunksize=10000, passes=20, eval_every = None, workers=1,\n",
    "                                     dtype=np.float64, iterations = 400)                                                                    \n",
    "    #set of relative important words in each topic corresponding to its keyName    \n",
    "    for i in range(0, n_topics):            #lda.num_topics - to view all topics generated by ldamodel\n",
    "        for word, prob in lda.show_topic(i,n_words):\n",
    "            themes_values.add(format(word))\n",
    "            themes[k] = themes_values       #creates a dictionary 'themes' where key=nodes of graph, value='themes_values'\n",
    "\n",
    "if debug==True:\n",
    "    with open('./ldathemes.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in themes:                  #to print the nodes and their corresponding values\n",
    "            outfile.write(key+':'+str(themes[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean themes already present in der_rel_form of the nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to remove themes that are already present in the synonyms of key node\n",
    "\n",
    "eg: synonyms: Obesity = corpulency, fleshiness, obesity\n",
    "      themes: Obesity = food, health, heart, fleshiness, eat, obesity\n",
    "   newthemes: Obesity = food, health, heart, eat\n",
    "   \"\"\" \n",
    "clean_themes = {}\n",
    "for k1 in themes.keys():\n",
    "    newtheme_values = set()\n",
    "    for k2 in der_rel_form.keys():\n",
    "        if(k1 == k2):\n",
    "            newtheme_values = (themes[k1].difference(der_rel_form[k1]))\n",
    "    clean_themes[k1] = newtheme_values\n",
    "\n",
    "if debug==True:\n",
    "    with open('./clean_themes.txt', 'w', encoding='utf8') as outfile: \n",
    "        for key in clean_themes: \n",
    "            outfile.write(key+':'+str(clean_themes[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](./img/pic4.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove non-entities from the cleaned themes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to get rid of non entities from the themes fetched\"\"\" \n",
    "\n",
    "#Create an Environment Variable[GOOGLE_APPLICATION_CREDENTIALS] which stores the API key value (used for authentication when making the API call)\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = \"./apikey.json\"\n",
    "client = language.LanguageServiceClient()\n",
    "#Now you can make the entity recognition call -\n",
    "new = {}\n",
    "for k1 in clean_themes.keys():\n",
    "    old = set()\n",
    "    for keywords in clean_themes[k1]:\n",
    "        document = language.types.Document(\n",
    "        content=keywords,\n",
    "        language='en',\n",
    "        type='PLAIN_TEXT'\n",
    "        )\n",
    "        #To make the API call \n",
    "        response = client.analyze_entities(document=document, encoding_type='UTF32')    \n",
    "        for entity in response.entities:\n",
    "            #entity.name fetches the recognized entities\n",
    "            old.add(entity.name)\n",
    "    new[k1] = old\n",
    "    time.sleep(100)\n",
    "\n",
    "if debug==True:\n",
    "    with open('./entity.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in new:\n",
    "            outfile.write(key+':'+str(new[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Absolute frequency of themes related keywords for each node of user map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to fetch ABSOLUTE FREQUENCY of theme related keywords from their corresponding relevant tweets set\"\"\" \n",
    "\n",
    "freq={}\n",
    "word = {}\n",
    "for k1 in clean_themes.keys():     \n",
    "    for k2 in nonEmtyRelevantTweets.keys():        \n",
    "        if(k1 == k2):\n",
    "            tweet_freq = {}\n",
    "            for keywords in clean_themes[k1]:                \n",
    "                count = 0\n",
    "                for tweet in nonEmtyRelevantTweets[k1]:                    \n",
    "                    if (' ' + keywords + ' ') in (' ' + tweet + ' '):\n",
    "                        count += 1\n",
    "                        tweet_freq[keywords] = count        \n",
    "        freq[k1] = tweet_freq\n",
    "\n",
    "if debug==True:\n",
    "    with open('./absolutefreq.txt', 'w', encoding='utf8') as outfile:\n",
    "        for key in freq:\n",
    "            outfile.write(key+':'+str(freq[key]))   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Most relevant themes related keywords using TF-IDF weight values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to fetch TF-IDF of most occuring keywords from relevant Tweets\"\"\" \n",
    "\n",
    "tweet_tfidf = {}\n",
    "for k1 in clean_themes.keys():\n",
    "    for k2 in nonEmtyRelevantTweets.keys():\n",
    "        if(k1 == k2):\n",
    "            tweet_weights = {}\n",
    "            cv = CountVectorizer()\n",
    "            # convert text data into term-frequency matrix\n",
    "            data = cv.fit_transform(nonEmtyRelevantTweets[k1])\n",
    "            tfidf_transformer = TfidfTransformer()\n",
    "            # convert term-frequency matrix into tf-idf\n",
    "            tfidf_matrix = tfidf_transformer.fit_transform(data)\n",
    "            # create dictionary to find a tfidf word each word\n",
    "            word2tfidf = dict(zip(cv.get_feature_names(), tfidf_transformer.idf_))\n",
    "            sorted_words = sorted(word2tfidf.items(), key=lambda x: x[1], reverse=False)\n",
    "            for word, score in sorted_words:\n",
    "                if word in clean_themes[k1]:\n",
    "                    if (score < max_df):\n",
    "                        tweet_weights[word] = (format(score))\n",
    "    tweet_tfidf[k1] = tweet_weights\n",
    "\n",
    "if debug==True:\n",
    "    with open('./tfidf.txt', 'w', encoding='utf8') as outfile:                    \n",
    "        for key in tweet_tfidf:                      \n",
    "            outfile.write(key+':'+str(tweet_tfidf[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confirm edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"to retrive final edges from comparison between TF-IDF and der_rel_form keywords\"\"\"\n",
    "\n",
    "final_edges = {}\n",
    "for k1, v1 in tweet_tfidf.items():\n",
    "    associated_edges = []\n",
    "    for v2 in v1.keys():                    #fetch the identified themes using tf-idf for those nodes\n",
    "        for k3, v3 in der_rel_form.items(): #fetch the corresponding synonyms of those nodes\n",
    "            if v2 in v3:                    #if themes 'v2' in 'synonyms' values\n",
    "                associated_edges.append(k3)\n",
    "    final_edges[k1] = associated_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"associated map\"\"\"\n",
    "\n",
    "associated_map=nx.DiGraph()\n",
    "for key in final_edges.keys(): \n",
    "    for z in range(0,len(final_edges[key])):\n",
    "        associated_map.add_edges_from([(str(key),str(final_edges[key][z]))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"confirmed map\"\"\"\n",
    "# can change size of plot for bigger maps display\n",
    "plt.rcParams['figure.figsize'] = [10, 7] # [width, height] of plot\n",
    "\n",
    "csv_filename = './confirmed_map.csv'\n",
    "\n",
    "for edge1 in usermap.edges():\n",
    "    usermap[edge1[0]][edge1[1]]['color'] = 'red'\n",
    "    \n",
    "count = 0\n",
    "\n",
    "with open(csv_filename, 'w') as f:\n",
    "    wtr = csv.writer(f, delimiter=',', lineterminator='\\n')\n",
    "    for edge1 in usermap.edges():\n",
    "        for edge2 in associated_map.edges():\n",
    "            if(edge1==edge2):\n",
    "                count += 1\n",
    "                wtr.writerow([edge1[0],edge1[1]])\n",
    "                usermap[edge2[0]][edge2[1]]['color'] = 'darkgreen'\n",
    "                edge_color_list = [ usermap[edge2[0]][edge2[1]]['color'] for edge2 in usermap.edges() ]\n",
    "f.close()            \n",
    "print(count)\n",
    "nx.draw_kamada_kawai(usermap, with_labels=True, font_size=15, node_color='lightblue',\n",
    "                     edge_color = edge_color_list, node_size=1000)\n",
    "plt.savefig(\"./confirmedmap.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adjacency matrix of confirmed map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#confirmed edges\n",
    "edge = list(set())\n",
    "for edge1 in usermap.edges():\n",
    "    edge.append([edge1[0],edge1[1]])\n",
    "    for edge2 in associated_map.edges():\n",
    "        if(edge1==edge2):\n",
    "            edge.append([edge2[0],edge2[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(edge)                                     #Create a datafarme\n",
    "df = pd.crosstab(df[0], df[1])                              #Compute a simple cross-tabulation of two (or more) factors\n",
    "# this step is to get the nodes with no relation as well\n",
    "idx = df.columns.union(df.index)                            #then reindex by union of column and index values \n",
    "df = df.reindex(index = idx, columns=idx, fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"save the adjacency matrix in a csv\n",
    "\n",
    "   0 = no relation, 1 = not confirmed edges, 2 = confirmed edges\n",
    "   From = rows, To = columns\n",
    "\"\"\"\n",
    "\n",
    "with open('./adjacencymatrix.csv', 'w') as fi:\n",
    "    df.to_csv(fi, header=True)\n",
    "fi.close()   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extend edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find association rules using PySpark 2.4.0 FP growth mlib with lift values\n",
    "# for code refer: https://spark.apache.org/docs/latest/api/python/pyspark.ml.html#module-pyspark.ml.fpm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import arules1  #convert rules into dictionary of antecedant and consequent and use it as a .py file to import here\n",
    "arules = arules1.arules #dictionary arules = {'antecedeant1':{'consequent1','consequent2'},'antecedeant2':{'consequent3'}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mapToRoot(word):\n",
    "    #this function takes a word e.g. 'obese' and maps it back to its root form e.g. obesity\n",
    "    for key in der_rel_form: #for each root form\n",
    "        if word in der_rel_form[key]:#if our input word is found within variations of this root form\n",
    "            return key#then we have the root form we wanted\n",
    "    \n",
    "#Given an association rule such as arules = {'obese':{'eat','strain','bias'},'dieting':{'slump'},'corona':{'feed','fleshy'},'depress':{'eater'}}\n",
    "#We INDEPENDENTLY MAP each of the words back to their root form\n",
    "def mapAssociationRules():\n",
    "    resultMap = {}\n",
    "    for key in arules:\n",
    "        resultMap[mapToRoot(key)]=list(set()) #create the entry, mapped\n",
    "        for consequent in arules[key]:\n",
    "            resultMap[mapToRoot(key)].append(mapToRoot(consequent)) #to the set we add the root form\n",
    "    return resultMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"arules confirmed/extended graph\"\"\"\n",
    "\n",
    "#removes keys with 'None' value\n",
    "arules_edges = dict((k, v) for k, v in mapAssociationRules().items() if k != None)\n",
    "#removes Value with 'None' value\n",
    "for k,v in arules_edges.items():\n",
    "    arules_val = []\n",
    "    for x in v:\n",
    "        if x != None:\n",
    "            arules_val.append(x)\n",
    "    arules_edges[k] = arules_val\n",
    "\n",
    "#create graph using arules_edges\n",
    "arules_map=nx.DiGraph()\n",
    "for key in arules_edges.keys(): \n",
    "    for z in range(0,len(arules_edges[key])):\n",
    "        arules_map.add_edges_from([(str(key),str(arules_edges[key][z]))])\n",
    "print(len(arules_map.edges()),('confirmed/extended edges using arules'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"extended map\"\"\"\n",
    "\n",
    "#find number of extended edges\n",
    "c=0\n",
    "for edge1 in arules_map.edges():\n",
    "    for edge2 in usermap.edges():\n",
    "        if(edge1==edge2):\n",
    "            c+=1\n",
    "print((len(arules_map.edges())-c),'extended edges using association rules')\n",
    "\n",
    "#find difference between usermap and arules_map to get extended edges\n",
    "arules_map_edges = set(arules_map.edges())\n",
    "usermap_edges = set(usermap.edges())\n",
    "extended_edges = list(arules_map_edges - usermap_edges)\n",
    "print(extended_edges)\n",
    "\n",
    "#create graph using extended_edges\n",
    "extended_map=nx.DiGraph()\n",
    "for edge in extended_edges: \n",
    "    extended_map.add_edges_from([edge])\n",
    "        \n",
    "#store the edges in a csv file\n",
    "csv_filename = './extended_map.csv'\n",
    "with open(csv_filename, 'w') as f:\n",
    "    wtr = csv.writer(f, delimiter=',', lineterminator='\\n')\n",
    "    for edge1 in extended_map.edges():\n",
    "        wtr.writerow([edge1[0],edge1[1]])\n",
    "f.close()  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}