Step3_Find_Influencers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "d55b22be",
    "execution_start": 1650738013901,
    "execution_millis": 1822,
    "cell_id": "7f90028b-e671-436d-aedb-c61883939c64",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 243
   },
   "source": "import numpy as np \nimport pandas as pd\nimport praw\nimport requests\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport pickle\nimport config\n#import networkx as nx\n",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "22adc8cc",
    "execution_start": 1649603148033,
    "execution_millis": 37,
    "cell_id": "00002-66272232-ec4e-498d-91bc-fa0000487742",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 135
   },
   "source": "#Setting up the Reddit API in python\nreddit = praw.Reddit(client_id=config.APP_ID,\n                     client_secret=config.APP_SECRET,\n                     user_agent=config.REDDIT_USERNAME)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": "## Get Recent Posts from Relevant Subreddits",
   "metadata": {
    "cell_id": "05d93fc7fd1d47c08e5596f819bb6c9a",
    "tags": [],
    "is_collapsed": false,
    "deepnote_cell_type": "text-cell-h2"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "323c1250d33b40aeb1e78eb6ac5e30d5",
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "20565fda",
    "execution_start": 1650746690120,
    "execution_millis": 1,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 621
   },
   "source": "def get_recent_posts(subred_list, n, max_com):\n    posts_info = []\n    sub_dict = {}\n    for subred in subred_list:\n        subreddit = reddit.subreddit(subred)\n        sub_dict[subreddit.id] = subred\n        print('gathering submissions and comments from',subred,'subreddit...')\n\n        for subm in subreddit.top(time_filter=\"month\", limit=n):\n            \n            subm.comment_sort = \"top\"\n            subm.comments.replace_more(limit=max_com) ##\n\n\n            subred_info = []\n            subred_info.append(subm.id)  \n            subred_info.append(str(subm.author)) \n            subred_info.append(subm.score)  \n            subred_info.append(subm.upvote_ratio)\n            subred_info.append(subm.num_comments)\n            subred_info.append(subred)\n            subred_info.append(datetime.fromtimestamp(subm.created_utc).strftime('%Y-%m-%dT%H:%M:%SZ'))\n            subred_info.append(subm.comments)\n            \n            posts_info.append(subred_info)\n        #print(len(posts_info), 'posts gathered so far...')\n    \n    sorted_info = sorted(posts_info, key=lambda x: x[1], reverse = True)\n    posts_df = pd.DataFrame(sorted_info, columns = ['id','author', 'score','upvote_ratio' ,'num_comments','subreddit','created','comments'])\n    print('completed gathering comments and submissions')\n    return sub_dict, posts_df",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": "## Get Influencers",
   "metadata": {
    "cell_id": "89149425-9560-42f2-8ad8-cbcdea08cd93",
    "tags": [],
    "is_collapsed": false,
    "deepnote_cell_type": "text-cell-h2"
   }
  },
  {
   "cell_type": "code",
   "metadata": {
    "tags": [],
    "cell_id": "00030-e6dea5e0-142f-4ebf-bc77-2e526788cb0e",
    "deepnote_to_be_reexecuted": false,
    "source_hash": "c33eba19",
    "execution_start": 1650738673036,
    "execution_millis": 44,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 3123
   },
   "source": "def get_influencers(subred_list, num_submissions, num_influencers, min_occur, mvp_flag, load_subm, orig_entry, max_com, equal_sub):\n    \n    '''\n    This function retrieves a ranked list of redditors (influencers) within relevant subreddits related \n    to the category of the search term. \n\n    Inputs: \n        subred_list - list of relevant subreddits\n        num_submission - number of submissions retrieved for each subreddit\n        num_influencers - number of influencers returned as a ranked list (highest to lowest)\n        min_occur - minimum number of times a redditor must appear in ranking list (comment or submission)\n        mvp_flag - indicates if we are running a pre-aggregated data set for our minimum viable product\n        load_subm - set to True if pickle file is available for posts versus retrieving through api\n        orig_entry - search term used for naming/finding pickle files\n        max_com - maximum comment request limit (32 each) for each submission (sorted by top)\n        equal_sub - if true, changes function to return an num_influencers from each subreddit in subred_list \n\n    Output: \n        influencer_lst - a list of influential authors of length = num_influencers\n    '''  \n    \n    orig_entry_mod = orig_entry.replace(' ','_')\n    file_name = './output_step3/influencers_' + orig_entry_mod + '.pickle'\n\n    if mvp_flag:\n        print('MVP mode - loading pickle file...')\n\n        #Load pickle file of relevant posts\n        try:\n            with open(file_name, 'rb') as f:\n                influencer_lst = pickle.load(f)\n            print('pickle file loaded.')\n        except:\n            print('Unable to find pickle file for ', orig_entry)\n\n    else:\n        #retrieve top n=num_submissions recent posts from each subreddit\n        #print('Retrieving posts...')\n        subm_filename = './output_step3/influencers_submissions_' + orig_entry_mod + '.pickle'\n        dict_filename = './output_step3/influencers_dict_' + orig_entry_mod + '.pickle'\n        if load_subm:\n            try:\n                with open(dict_filename, 'rb') as f:\n                    sub_dict = pickle.load(f)\n                subm_df = pd.read_pickle(subm_filename)  \n\n            except:\n                print('Unable to find pickle file for ', orig_entry) \n        \n        else:\n            sub_dict, subm_df = get_recent_posts(subred_list, num_submissions,max_com)\n            #with open (dict_filename, 'wb') as f:\n            #    pickle.dump(sub_dict, f)\n            #subm_df.to_pickle(subm_filename)\n\n        \n        submission_list = list(subm_df['id'])\n        \n        #normalize submission scoring by total of score for submissions\n        rank_subm = subm_df[['id','score','author','subreddit']]\n        rank_subm = rank_subm.assign(type='submission')\n        rank_subm_sum = rank_subm.score.sum()\n        rank_subm['score'] = rank_subm['score']/rank_subm_sum\n\n        #retrieve comments from submission_list\n        com_info = []\n        for com_list in subm_df['comments']:\n            for com in com_list:\n                com_info.append([com.id,com.score,str(com.author),com.body,com.subreddit_id,com.parent_id,'comment'])\n\n        com_df = pd.DataFrame(com_info, columns=['id','score','author','body','subreddit_id','parent_id','type'])\n        \n        #print('Retrieving comments...')\n        #com_df = get_comments(submission_list)\n        #print('Done...',len(com_df),' comments retrieved.')\n        \n        sub_map = {'t5_' + str(key): val for key, val in sub_dict.items()}\n        com_df['subreddit'] = com_df['subreddit_id'].map(sub_map)\n\n        #filter out comments with no author or comments that have been removed or deleted      \n        com_df = com_df[com_df['author'] != 'None']\n        com_df = com_df[com_df['author'] != 'AutoModerator']\n        com_df = com_df[~com_df['body'].isin(['[removed]','[deleted]'])]\n        \n        #normalize comment scoring by total of score for comments\n        rank_com = com_df[['id','score','author','subreddit','type']].copy()\n        rank_com_sum = rank_com.score.sum()\n        rank_com['score'] = rank_com['score']/rank_com_sum\n\n        #combined normalized rankings for submissions and comments\n        rank_df = pd.concat([rank_com, rank_subm], axis=0)\n\n        #aggregate submissiona and comment scores by author\n        rank_df_agg = pd.pivot_table(rank_df,index=['author'], aggfunc={'score': np.sum, 'id': len}).rename(columns={'id': 'count'})\n        rank_df_agg = rank_df_agg[rank_df_agg['count'] > min_occur].sort_values(by='score',ascending=False)\n        \n        #take top 25 scoring authors to include in influencer list\n        author_lst_top25_score = list(rank_df_agg.index[:25])\n        author_df_top250 = rank_df_agg[:250].copy()\n               \n        print('collecting comment karma for top 250 authors...')\n        karma_list = []\n        for author in list(author_df_top250.index):\n            try:\n                karma_list.append(reddit.redditor(author).comment_karma)\n            except:\n                karma_list.append(0)\n\n        author_df_top250['karma'] = karma_list\n        karma_df = author_df_top250.copy()\n        karma_df = karma_df[~karma_df.index.isin(author_lst_top25_score)]\n        karma_df = karma_df.sort_values(by='karma', ascending=False)\n\n        author_lst_top50_karma = list(karma_df.index[:50])\n\n\n        influencer_lst = author_lst_top25_score + author_lst_top50_karma\n        #print(influencer_lst)\n        #print(len(influencer_lst),'influencers found so far...')\n\n        if equal_sub:\n            #filter scoring for each subreddit\n            #author_lst = []\n            for sub in subred_list:\n                #aggregate scoring for both comments and submissions by author and rank\n                author_df = rank_df[rank_df['subreddit'] == sub]\n                if len(author_df) > 0:\n                    author_df = pd.pivot_table(author_df,index=['author'], aggfunc={'score': np.sum, 'id': len}).rename(columns={'id': 'count'})\n                    author_df = author_df[~author_df.index.isin(influencer_lst)]\n                    author_df = author_df[author_df['count'] > min_occur].sort_values(by='score',ascending=False)\n                    influencer_lst = influencer_lst + list(author_df.index)[:num_influencers] \n                    #author_lst.append(list(author_df.index)[:num_influencers])\n                    #print('Top',num_influencers,'influencers added from',sub,'subreddit.',len(influencer_lst),'found so far.')\n                else:\n                    continue\n            #author_lst = [item for sublist in author_lst for item in sublist]\n\n        else:\n            \n            \n            #aggregate scoring for both comments and submissions by author and rank\n            author_df = pd.pivot_table(rank_df,index=['author'], aggfunc={'score': np.sum, 'id': len}).rename(columns={'id': 'count'})\n            author_df = author_df[author_df['count'] > min_occur].sort_values(by='score',ascending=False)\n            \n            #retrieve list of top n authors\n            author_lst = list(author_df.index)\n            author_lst = author_lst[:num_influencers]\n\n        #influencer_lst = influencer_lst + author_lst \n        inf_com_df = com_df[com_df['author'].isin(influencer_lst)]\n        #Create pickle file from our list of influencers\n            \n        with open (file_name, 'wb') as f:\n            pickle.dump(influencer_lst, f)\n        \n\n\n        ## save dataframes for analysis\n        #df_list = [inf_com_df, author_df, com_df, rank_com, rank_subm, author_df_top250]\n        #df_names = ['inf_com_df', 'author_df', 'com_df', 'rank_com', 'rank_subm','author_df_top250']\n        #count = 0\n\n        #for df in df_list:\n            #df.name = df_names[count] \n            #file_name_df = './output_step3/'+ df.name + '_' + orig_entry_mod + '.pickle'\n            #df.to_pickle(file_name_df)\n            #count += 1\n\n        print('pickle file of influencers has been created with',len(set(influencer_lst)),'redditors')\n    return influencer_lst",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "f9911eb0e8c446ec82c5867e1d509a48",
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "ea968632",
    "execution_start": 1649909007268,
    "execution_millis": 1,
    "owner_user_id": "f9c4ad17-97d3-4f24-b7b0-bc099d48c5ec",
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 171
   },
   "source": "#num_submissions = 50\n#num_influencers = 5\n#min_occur = 2\n#max_com = 0\n#subred_list = []\n#search_term = 'Elon Musk'",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "cell_id": "f1e6d698c9814292895fb9c9beb1131d",
    "tags": [],
    "deepnote_to_be_reexecuted": false,
    "source_hash": "130edf87",
    "execution_start": 1649909007270,
    "execution_millis": 89933,
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 81
   },
   "source": "#influencer_lst = get_influencers(subred_list, num_submissions, num_influencers, min_occur, True, False,search_term, max_com,True)",
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": "",
   "metadata": {
    "cell_id": "4ba9ce7750624dffa87a91904345df61",
    "tags": [],
    "deepnote_cell_type": "code",
    "deepnote_cell_height": 66
   },
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": "<a style='text-decoration:none;line-height:16px;display:flex;color:#5B5B62;padding:10px;justify-content:end;' href='https://deepnote.com?utm_source=created-in-deepnote-cell&projectId=90b052a7-f47d-474e-888f-9345355cfd9a' target=\"_blank\">\n<img alt='Created in deepnote.com' style='display:inline;max-height:16px;margin:0px;margin-right:7.5px;' src='data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyB3aWR0aD0iODBweCIgaGVpZ2h0PSI4MHB4IiB2aWV3Qm94PSIwIDAgODAgODAiIHZlcnNpb249IjEuMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayI+CiAgICA8IS0tIEdlbmVyYXRvcjogU2tldGNoIDU0LjEgKDc2NDkwKSAtIGh0dHBzOi8vc2tldGNoYXBwLmNvbSAtLT4KICAgIDx0aXRsZT5Hcm91cCAzPC90aXRsZT4KICAgIDxkZXNjPkNyZWF0ZWQgd2l0aCBTa2V0Y2guPC9kZXNjPgogICAgPGcgaWQ9IkxhbmRpbmciIHN0cm9rZT0ibm9uZSIgc3Ryb2tlLXdpZHRoPSIxIiBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPgogICAgICAgIDxnIGlkPSJBcnRib2FyZCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTEyMzUuMDAwMDAwLCAtNzkuMDAwMDAwKSI+CiAgICAgICAgICAgIDxnIGlkPSJHcm91cC0zIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgxMjM1LjAwMDAwMCwgNzkuMDAwMDAwKSI+CiAgICAgICAgICAgICAgICA8cG9seWdvbiBpZD0iUGF0aC0yMCIgZmlsbD0iIzAyNjVCNCIgcG9pbnRzPSIyLjM3NjIzNzYyIDgwIDM4LjA0NzY2NjcgODAgNTcuODIxNzgyMiA3My44MDU3NTkyIDU3LjgyMTc4MjIgMzIuNzU5MjczOSAzOS4xNDAyMjc4IDMxLjY4MzE2ODMiPjwvcG9seWdvbj4KICAgICAgICAgICAgICAgIDxwYXRoIGQ9Ik0zNS4wMDc3MTgsODAgQzQyLjkwNjIwMDcsNzYuNDU0OTM1OCA0Ny41NjQ5MTY3LDcxLjU0MjI2NzEgNDguOTgzODY2LDY1LjI2MTk5MzkgQzUxLjExMjI4OTksNTUuODQxNTg0MiA0MS42NzcxNzk1LDQ5LjIxMjIyODQgMjUuNjIzOTg0Niw0OS4yMTIyMjg0IEMyNS40ODQ5Mjg5LDQ5LjEyNjg0NDggMjkuODI2MTI5Niw0My4yODM4MjQ4IDM4LjY0NzU4NjksMzEuNjgzMTY4MyBMNzIuODcxMjg3MSwzMi41NTQ0MjUgTDY1LjI4MDk3Myw2Ny42NzYzNDIxIEw1MS4xMTIyODk5LDc3LjM3NjE0NCBMMzUuMDA3NzE4LDgwIFoiIGlkPSJQYXRoLTIyIiBmaWxsPSIjMDAyODY4Ij48L3BhdGg+CiAgICAgICAgICAgICAgICA8cGF0aCBkPSJNMCwzNy43MzA0NDA1IEwyNy4xMTQ1MzcsMC4yNTcxMTE0MzYgQzYyLjM3MTUxMjMsLTEuOTkwNzE3MDEgODAsMTAuNTAwMzkyNyA4MCwzNy43MzA0NDA1IEM4MCw2NC45NjA0ODgyIDY0Ljc3NjUwMzgsNzkuMDUwMzQxNCAzNC4zMjk1MTEzLDgwIEM0Ny4wNTUzNDg5LDc3LjU2NzA4MDggNTMuNDE4MjY3Nyw3MC4zMTM2MTAzIDUzLjQxODI2NzcsNTguMjM5NTg4NSBDNTMuNDE4MjY3Nyw0MC4xMjg1NTU3IDM2LjMwMzk1NDQsMzcuNzMwNDQwNSAyNS4yMjc0MTcsMzcuNzMwNDQwNSBDMTcuODQzMDU4NiwzNy43MzA0NDA1IDkuNDMzOTE5NjYsMzcuNzMwNDQwNSAwLDM3LjczMDQ0MDUgWiIgaWQ9IlBhdGgtMTkiIGZpbGw9IiMzNzkzRUYiPjwvcGF0aD4KICAgICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgIDwvZz4KPC9zdmc+' > </img>\nCreated in <span style='font-weight:600;margin-left:4px;'>Deepnote</span></a>",
   "metadata": {
    "tags": [],
    "created_in_deepnote_cell": true,
    "deepnote_cell_type": "markdown"
   }
  }
 ],
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "orig_nbformat": 2,
  "deepnote": {
   "is_reactive": false
  },
  "deepnote_notebook_id": "8cd8d12b-fd60-414c-86ba-86b2816db824",
  "deepnote_execution_queue": []
 }
}