profile.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "459ff43d-d46f-4b57-8543-b38c7a93d03a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b263335c-d769-49d2-bfce-ec2d00270e91",
   "metadata": {},
   "outputs": [],
   "source": [
    "caps = DesiredCapabilities.CHROME\n",
    "caps['goog:loggingPrefs'] = {'performance': 'ALL'}\n",
    "\n",
    "options = webdriver.ChromeOptions()\n",
    "options.binary_location = 'C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\\\brave.exe'\n",
    "# caps['binary_location'] = 'C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\\\brave.exe'\n",
    "\n",
    "driver = webdriver.Chrome(desired_capabilities=caps, options=options)\n",
    "driver.get('https://twitter.com/bps_statistics')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f87e7ccb-4401-46ff-9c7a-63b641b0bdbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_browser_log_entry(entry):\n",
    "    response = json.loads(entry['message'])['message']\n",
    "    return response\n",
    "\n",
    "def process_network_response_body(entry):\n",
    "    response = json.loads(entry['body'])\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "129d8e5a-8d40-4fcd-a5ad-826dbb410812",
   "metadata": {},
   "outputs": [],
   "source": [
    "browser_log = driver.get_log('performance') \n",
    "events = [process_browser_log_entry(entry) for entry in browser_log]\n",
    "events = [event for event in events if 'Network.response' in event['method']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8b6ddec1-b374-44cb-a39d-287357a00500",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = [event for event in events if (('Network.response' in event['method']) & (('UserByScreenName' in event['params'].get('response', {}).get('url', '')) | (('UserTweets' in event['params'].get('response', {}).get('url', '')))))][0]['params']['response']['url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "60799d21-3903-49c5-b08b-9e21faa2a6bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'7mjxD3-C6BxitPMVQ6w0-Q'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url.split('/')[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "d11702dc-acea-428e-8788-f09aa7f28a2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.parse import quote, unquote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "0e87d756-2e07-457c-b58d-b017ab50e239",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"userId\":\"{userid}\",\"count\":40,\"includePromotedContent\":true,\"withQuickPromoteEligibilityTweetFields\":true,\"withSuperFollowsUserFields\":true,\"withDownvotePerspective\":false,\"withReactionsMetadata\":false,\"withReactionsPerspective\":false,\"withSuperFollowsTweetFields\":true,\"withVoice\":true,\"withV2Timeline\":false,\"__fs_dont_mention_me_view_api_enabled\":false,\"__fs_interactive_text_enabled\":false,\"__fs_responsive_web_uc_gql_enabled\":false}'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unquote('%7B%22userId%22%3A%22{userid}%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%2C%22__fs_interactive_text_enabled%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%7D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34344567-256e-4a19-8a8c-e52853e8f027",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}