Pubmed_Clinical_Trials_data_extraction_1.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/elrf3lipes/Python_Automation_Projects/blob/master/Pubmed_Clinical_Trials_data_extraction_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b02gtzI_PZNG"
      },
      "source": [
        "# PubMed - Search query\n",
        "Enter a search expression, email address and date-range(YYYY/MM/DD), get total number of publications and 10 related PMIDs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "hDuD9mqfOgii"
      },
      "outputs": [],
      "source": [
        "# @title\n",
        "!pip install Bio\n",
        "\n",
        "from Bio import Entrez\n",
        "from Bio import Medline\n",
        "from datetime import date\n",
        "\n",
        "MAX_COUNT = 10\n",
        "TERM = input(\"Type your search term: \")\n",
        "\n",
        "print('Getting {0} publications containing {1}...'.format(MAX_COUNT, TERM))\n",
        "Entrez.email = input(\"Enter your email: \")\n",
        "h = Entrez.esearch(db='pubmed', retmax=MAX_COUNT, term=TERM, mindate = input(\"Start date: \"),  maxdate = input(\"End date: \"), datetype=\"pdat\")\n",
        "result = Entrez.read(h)\n",
        "print('Total number of publications containing {0}: {1}'.format(TERM, result['Count']))\n",
        "ids = result['IdList']\n",
        "h = Entrez.efetch(db='pubmed', id=ids, rettype='medline', retmode='text')\n",
        "records = Medline.parse(h)\n",
        "\n",
        "# Print each ID one per line\n",
        "for article_id in ids:\n",
        "    print(article_id)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NGccqsI-vQ2q"
      },
      "source": [
        "# Search for PMID affiliations\n",
        "Enter PMID, get all the affiliations listed"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z6FzXsiQS0i9",
        "outputId": "8276cf39-4f12-4107-d879-cb945a93a7b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter the article ID: 20301790\n",
            "Affiliations:\n",
            "1. Department of Pediatrics, Amalia Children’s Hospital, Radboud University Medical Center, Nijmegen, the Netherlands\n",
            "2. Department of Neurology, Donders Institute for Brain, Cognition and Behaviour;, Radboud University Medical Center, Nijmegen, the Netherlands\n",
            "3. Amalia Children's Hospital, Radboud University Medical Center, Nijmegen, the Netherlands\n",
            "4. Department of Human Genetics, Radboud University Medical Center, Nijmegen, the Netherlands\n",
            "5. Department of Pediatric Neurology, Donders Institute for Brain, Cognition and Behaviour;, Amalia Children's Hospital, Radboud University Medical Center, Nijmegen, the Netherlands\n"
          ]
        }
      ],
      "source": [
        "# @title\n",
        "import xml.etree.ElementTree as ET\n",
        "from urllib.request import urlopen\n",
        "\n",
        "# Get the article ID from the user\n",
        "article_id = input(\"Enter the article ID: \")\n",
        "\n",
        "# Construct the efetch URL using the user's input\n",
        "efetch_url = \"http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?&db=pubmed&retmode=xml&id=%s\" % (article_id)\n",
        "\n",
        "# Fetch and parse the XML data\n",
        "response = urlopen(efetch_url)\n",
        "xml_data = response.read()\n",
        "response.close()\n",
        "\n",
        "# Parse the XML data\n",
        "root = ET.fromstring(xml_data)\n",
        "\n",
        "# Find and print affiliations\n",
        "affiliations = []\n",
        "\n",
        "# Look for Affiliation elements within Author elements\n",
        "for author in root.findall(\".//Author\"):\n",
        "    affiliation = author.find(\".//Affiliation\")\n",
        "    if affiliation is not None:\n",
        "        affiliations.append(affiliation.text.strip())\n",
        "\n",
        "# Print affiliations, if any were found\n",
        "if affiliations:\n",
        "    print(\"Affiliations:\")\n",
        "    for i, affiliation in enumerate(affiliations, 1):\n",
        "        print(f\"{i}. {affiliation}\")\n",
        "else:\n",
        "    print(\"No affiliations found for this article.\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5DOXErdODCi"
      },
      "source": [
        "# Affiliations parser\n",
        "Enter PMID, get the simplified afiliation names collapsed and their counts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 192
        },
        "id": "RmNozxGmIjUV",
        "outputId": "d813beca-0a38-43e2-fd8c-34d23e3c1955",
        "cellView": "form"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter the article ID: 20301790\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7ec1cf64f2e0>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "</style>\n",
              "<table id=\"T_f15da\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th id=\"T_f15da_level0_col0\" class=\"col_heading level0 col0\" >Affiliation</th>\n",
              "      <th id=\"T_f15da_level0_col1\" class=\"col_heading level0 col1\" >Count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_f15da_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
              "      <td id=\"T_f15da_row0_col0\" class=\"data row0 col0\" >Amalia Children’s Hospital</td>\n",
              "      <td id=\"T_f15da_row0_col1\" class=\"data row0 col1\" >1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f15da_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
              "      <td id=\"T_f15da_row1_col0\" class=\"data row1 col0\" >Donders Institute for Brain</td>\n",
              "      <td id=\"T_f15da_row1_col1\" class=\"data row1 col1\" >2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f15da_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
              "      <td id=\"T_f15da_row2_col0\" class=\"data row2 col0\" >Amalia Children's Hospital</td>\n",
              "      <td id=\"T_f15da_row2_col1\" class=\"data row2 col1\" >1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_f15da_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
              "      <td id=\"T_f15da_row3_col0\" class=\"data row3 col0\" >Radboud University Medical Center</td>\n",
              "      <td id=\"T_f15da_row3_col1\" class=\"data row3 col1\" >1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "# @title\n",
        "import xml.etree.ElementTree as ET\n",
        "from urllib.request import urlopen\n",
        "import re\n",
        "import pandas as pd\n",
        "from IPython.display import display\n",
        "\n",
        "# Get the article ID from the user\n",
        "article_id = input(\"Enter the article ID: \")\n",
        "\n",
        "# Construct the efetch URL using the user's input\n",
        "efetch_url = \"http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?&db=pubmed&retmode=xml&id=%s\" % (article_id)\n",
        "\n",
        "# Fetch and parse the XML data\n",
        "response = urlopen(efetch_url)\n",
        "xml_data = response.read()\n",
        "response.close()\n",
        "\n",
        "# Parse the XML data\n",
        "root = ET.fromstring(xml_data)\n",
        "\n",
        "# Find and print affiliations\n",
        "affiliations = []\n",
        "\n",
        "# Look for Affiliation elements within Author elements\n",
        "for author in root.findall(\".//Author\"):\n",
        "    affiliation = author.find(\".//Affiliation\")\n",
        "    if affiliation is not None:\n",
        "        affiliations.append(affiliation.text.strip())\n",
        "\n",
        "def extract_phrases_and_countries(affiliations):\n",
        "    # Initialize a dictionary to store the counts of extracted phrases\n",
        "    extracted_phrases = {}\n",
        "\n",
        "    # Define a regular expression pattern to match the specified words\n",
        "    word_pattern = r'\\b(?:Hospital|University|Institute|School|Academy|Clinic)\\b'\n",
        "\n",
        "    for affiliation in affiliations:\n",
        "        # Split the affiliation into sentences using '.'\n",
        "        sentences = affiliation.split('.')\n",
        "\n",
        "        for sentence in sentences:\n",
        "            # Initialize a flag to check if a group has already been found in the current sentence\n",
        "            group_found = False\n",
        "\n",
        "            # Split the sentence into groups using ','\n",
        "            groups = re.split(r',', sentence)\n",
        "\n",
        "            for group in groups:\n",
        "                if re.search(word_pattern, group) and not group_found:\n",
        "                    # Use the stripped group as the key in the dictionary\n",
        "                    group_key = group.strip()\n",
        "                    # If the key exists, increment the count; otherwise, initialize it to 1\n",
        "                    if group_key in extracted_phrases:\n",
        "                        extracted_phrases[group_key] += 1\n",
        "                    else:\n",
        "                        extracted_phrases[group_key] = 1\n",
        "                    group_found = True\n",
        "\n",
        "    # Create lists for the DataFrame\n",
        "    phrases = []\n",
        "    counts = []\n",
        "\n",
        "    # Iterate over the keys and values in the extracted_phrases dictionary\n",
        "    for phrase, count in extracted_phrases.items():\n",
        "        phrases.append(phrase)\n",
        "        counts.append(count)\n",
        "\n",
        "    # Create a DataFrame\n",
        "    df = pd.DataFrame({\n",
        "        'Affiliation': phrases,\n",
        "        'Count': counts,\n",
        "    })\n",
        "\n",
        "    return df\n",
        "\n",
        "result_df = extract_phrases_and_countries(affiliations)\n",
        "\n",
        "# Format and style the DataFrame\n",
        "styled_df = result_df.style.format(precision=3, thousands=\".\", decimal=\",\")\n",
        "\n",
        "# Display the styled DataFrame\n",
        "display(styled_df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G-lLB64qPvxq"
      },
      "source": [
        "# ClinicalTrials - Search query\n",
        "Enter a search expression, get total number of publications and 10 related NTCIds with their respective affiliations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "s7bz_hm0P3se"
      },
      "outputs": [],
      "source": [
        "# @title\n",
        "!pip install pytrials\n",
        "\n",
        "from pytrials.client import ClinicalTrials\n",
        "import pandas as pd\n",
        "\n",
        "# Prompt the user for the search expression\n",
        "search_expr = input(\"Enter your search expression: \")\n",
        "\n",
        "# Initialize the ClinicalTrials client\n",
        "ct = ClinicalTrials()\n",
        "\n",
        "# Get 50 full studies related to the user-provided search expression in json format.\n",
        "full_studies = ct.get_full_studies(search_expr=search_expr, max_studies=10)\n",
        "\n",
        "# Get the NCTId, Condition, and Brief title fields from 50 studies related to the user-provided search expression, in csv format.\n",
        "corona_fields = ct.get_study_fields(\n",
        "    search_expr=search_expr,\n",
        "    fields=[\"NCTId\", \"Condition\", \"StudyFirstPostDate\", \"CompletionDate\", \"OrgFullName\"],\n",
        "    max_studies=10,\n",
        "    fmt=\"csv\",\n",
        ")\n",
        "\n",
        "# Get the count of studies related to the user-provided search expression.\n",
        "study_count = ct.get_study_count(search_expr=search_expr)\n",
        "\n",
        "# Display the count of studies\n",
        "print(f\"Total studies related to '{search_expr}': {study_count}\")\n",
        "\n",
        "# Read the csv data in Pandas\n",
        "df = pd.DataFrame.from_records(corona_fields[1:], columns=corona_fields[0])\n",
        "\n",
        "# Apply styling to the DataFrame for formatting\n",
        "styled_df = df.style \\\n",
        "    .format(precision=3, thousands=\".\", decimal=\",\") \\\n",
        "    .set_table_styles([{'selector': 'th', 'props': [('font-size', '12px')]}])  # You can customize styling as needed\n",
        "\n",
        "# Display the styled DataFrame\n",
        "styled_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ExFQNTIJSsjQ"
      },
      "source": [
        "# Next-steps:\n",
        "Produce report -\n",
        "Export CSV or other file types"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMmV/7tiarMzdHkoHa/cgpY",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}