analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DA-DAN Modulation Analysis Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set-Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### imports\n",
    "import navis\n",
    "import fafbseg\n",
    "import flybrains\n",
    "\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from functools import reduce\n",
    "from tabulate import tabulate\n",
    "import hvplot.pandas\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from matplotlib.colors import rgb2hex, to_rgb\n",
    "import matplotlib.gridspec as gridspec\n",
    "\n",
    "import pyroglancer\n",
    "from pyroglancer.localserver import startdataserver, closedataserver\n",
    "from pyroglancer.flywire import flywireurl2dict, add_flywirelayer, set_flywireviewerstate\n",
    "\n",
    "import navis.interfaces.neuprint as neu\n",
    "from navis.interfaces.neuprint import NeuronCriteria as NC, SynapseCriteria as SC\n",
    "from navis.interfaces.neuprint import fetch_adjacencies, fetch_synapse_connections\n",
    "\n",
    "from pyroglancer.layers import create_nglayer, setlayerproperty\n",
    "from pyroglancer.ngviewer import openviewer, closeviewer,setviewerstate, get_ngscreenshot\n",
    "from pyroglancer.ngspaces import create_ngspace\n",
    "from pyroglancer.createconfig import createconfig\n",
    "\n",
    "from neuprint import fetch_synapses, NeuronCriteria as NC, SynapseCriteria as SC\n",
    "\n",
    "\n",
    "import re\n",
    "\n",
    "import utils\n",
    "from utils import loadConnections, extractOutputsPerType, extractInputsPerType, plotPAMStatistic, loadPickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from pandas.errors import SettingWithCopyWarning\n",
    "#warnings.simplefilter(action=\"ignore\", category=SettingWithCopyWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "label_fontsize = 16 \n",
    "title_fontsize = 18\n",
    "\n",
    "figwidth_size = 5\n",
    "figheigth_size = 5\n",
    "\n",
    "#navis display options for frontal view..\n",
    "elev = -180\n",
    "azim = -90\n",
    "dist = 6\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('navis version :',navis.__version__)\n",
    "print('fafbseg version :',fafbseg.__version__)\n",
    "print('flybrains version :',flybrains.__version__)\n",
    "print('pyroglancer version :',pyroglancer.__version__)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetch data from Neuprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## setting up Neuprint Client\n",
    "## using dotenv to import Janelia PAT\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "load_dotenv()\n",
    "\n",
    "## fetching Janelia client\n",
    "client = neu.Client('https://neuprint.janelia.org/', dataset='hemibrain:v1.2.1', token=os.environ.get(\"JANELIA_PAT\"))\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### print roi hierarchy\n",
    "rois = neu.fetch_roi_hierarchy(False,True,'text')\n",
    "print(rois)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get PAM neurons from hemibrains and fetch their connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## fetch all neurons containing \"PAM\" from hemibrain dataset\n",
    "pamneurons_df, roi_counts_df = neu.fetch_neurons(NC(status='Traced',type=\"^PAM.*\",regex=True)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### print unique PAM types\n",
    "uniquePAMTypes = pamneurons_df.type.unique()\n",
    "print(uniquePAMTypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### get PAM-PAM connections\n",
    "PAM_PAM_Connections = loadConnections(silent=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### extract unique neuron instances from PAM PAM connections and prepare for dict\n",
    "uniquePAMNeuronsPre = PAM_PAM_Connections[\"instance_pre\"].unique()\n",
    "print(uniquePAMNeuronsPre)\n",
    "uniquePAMNeuronsPost = PAM_PAM_Connections[\"instance_post\"].unique()\n",
    "print(\"\")\n",
    "print(uniquePAMNeuronsPost)\n",
    "\n",
    "# Parse the unique values into a string for a dict\n",
    "possibleTargetsInstance = \"{\" + \",\\n\".join(f\"'{neuron}': '{neuron}'\" for neuron in sorted(set(uniquePAMNeuronsPre).union(uniquePAMNeuronsPost))) + \"}\"\n",
    "print(\"\\nPossible Targets Instance for dict:\")\n",
    "print(possibleTargetsInstance)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Heatmap Visualization of PAM-PAM connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAMSuperTypeCollapsedConnections = PAM_PAM_Connections.replace(to_replace=r'PAM(\\d{2})\\_?\\w?', value=r'PAM', regex=True)\n",
    "\n",
    "#### visualize as matrix\n",
    "matrix = neu.connection_table_to_matrix(PAMSuperTypeCollapsedConnections, 'type', sort_by='type',)\n",
    "    ### note: this originally threw an error bc of deprecated call to df.pivot(), fixed it by updating the pivot call in neuprint/utils.py to:\n",
    "    ### matrix = agg_weights_df.pivot(index=col_pre, columns=col_post, values=weight_col)\n",
    "\n",
    "matrix.index = matrix.index.astype(str)\n",
    "matrix.columns = matrix.columns.astype(str)\n",
    "\n",
    "title=\"PAM-PAM Connections Heatmap\"\n",
    "matrix.hvplot.heatmap(height=600, width=700, xaxis='top', title=title).opts(xrotation=60)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### visualize as matrix\n",
    "matrix = neu.connection_table_to_matrix(PAM_PAM_Connections, 'type', sort_by='type',)\n",
    "    ### note: this originally threw an error bc of deprecated call to df.pivot(), fixed it by updating the pivot call in neuprint/utils.py to:\n",
    "    ### matrix = agg_weights_df.pivot(index=col_pre, columns=col_post, values=weight_col)\n",
    "\n",
    "matrix.index = matrix.index.astype(str)\n",
    "matrix.columns = matrix.columns.astype(str)\n",
    "\n",
    "title=\"PAM-PAM Connections Heatmap\"\n",
    "matrix.hvplot.heatmap(height=600, width=700, xaxis='top', title=title).opts(xrotation=60)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### collapse PAM types and then show as matrix\n",
    "\n",
    "PAMCollapsedConnections = PAM_PAM_Connections.replace(to_replace=r'PAM(\\d{2})\\_?\\w?', value=r'PAM\\1', regex=True)\n",
    "\n",
    "matrix = neu.connection_table_to_matrix(PAMCollapsedConnections, 'type', sort_by='type',)\n",
    "    ### note: this originally threw an error bc of deprecated call to df.pivot(), fixed it by updating the pivot call in neuprint/utils.py to:\n",
    "    ### matrix = agg_weights_df.pivot(index=col_pre, columns=col_post, values=weight_col)\n",
    "\n",
    "matrix.index = matrix.index.astype(str)\n",
    "matrix.columns = matrix.columns.astype(str)\n",
    "\n",
    "matrix.hvplot.heatmap(height=600, width=700, xaxis='top', title = title+\" -- PAM Types Collapsed\").opts(xrotation=60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### show unique neurons as matrix\n",
    "matrix = neu.connection_table_to_matrix(PAM_PAM_Connections, 'instance', sort_by='instance',)\n",
    "    ### note: this originally threw an error bc of deprecated call to df.pivot(), fixed it by updating the pivot call in neuprint/utils.py to:\n",
    "    ### matrix = agg_weights_df.pivot(index=col_pre, columns=col_post, values=weight_col)\n",
    "\n",
    "matrix.index = matrix.index.astype(str)\n",
    "matrix.columns = matrix.columns.astype(str)\n",
    "\n",
    "title=\"PAM-PAM Connections Heatmap\"\n",
    "matrix.hvplot.heatmap(height=600, width=700, xaxis='top', title=title+\" -- PAM neurons\").opts(xrotation=60)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### detailed connection analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prints out unique pre- and postsynaptic connection partners (grouped by supertype, type, subtype, instance, and bodyID) of a specific PAM type or all PAMs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Printing unique connection partners of PAM04 neurons\")\n",
    "subTypePartners, instancePartners, bodyIdPartners = utils.listUniqueConnectionPartners(extractInputsPerType(\"PAM04\",connections=PAM_PAM_Connections))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### retrieving all connections between PAM neurons and any neurons\n",
    "PAM_All_Connections = loadConnections(\"^PAM.*\",\"^.*\", bidirectional=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAM_All_Connections['weight'].plot(kind='hist', bins=201, title='Frequency of Connections by Weight', logy=True)\n",
    "# Calculate the fifth percentile threshold value for the 'weight' column\n",
    "fifth_percentile_threshold = PAM_All_Connections['weight'].quantile(0.05)\n",
    "print(f\"Fifth percentile threshold value for connection weights: {fifth_percentile_threshold}\")\n",
    "\n",
    "plt.xlabel('Weight')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Printing out unique connection partners of all PAMs\")\n",
    "print(\"\\n Presynaptic\")\n",
    "typePartners, instancePartners, bodyIdPartners = utils.listUniqueConnectionPartners(PAM_All_Connections,type=\"pre\")\n",
    "print(\"\\n Postsynaptic\")\n",
    "typePartners, instancePartners, bodyIdPartners = utils.listUniqueConnectionPartners(PAM_All_Connections,type=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##testing whether connections make sense\n",
    "extractInputsPerType(target=\"PAM05\",connections=PAM_All_Connections)\n",
    "extractOutputsPerType(target=\"PAM05\",connections=PAM_All_Connections)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### \n",
    "targets = [\"PAM01\",\"PAM02\",\"PAM03\",\"PAM04\"]\n",
    "partnerType = \"type\" ## instance, type, bodyId\n",
    "#connections = filteredPAMConnections\n",
    "connections = PAM_PAM_Connections\n",
    "treshhold= 0.005\n",
    "#plotPAMStatistic(connections=connections,targets=targets,etcTreshhold=treshhold,partnerMode=partnerType,normalized=False, mergePAMSubtypes = True, title=\"All PAM synapses statistic\", mergePAMsupertype=  False, weightFilterThreshhold=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### \n",
    "targets = [\"PAM01_a(y5)_L\"]\n",
    "partnerType = \"type\" ## instance, type, bodyId\n",
    "\n",
    "treshhold= 0.02\n",
    "#plotPAMStatistic(connections=filteredPAMConnections,targets=targets,targetMode=\"instance\",etcTreshhold=treshhold,partnerMode=partnerType,normalized=False, mergePAMSubtypes = False, title=\"All PAM synapses statistic\", mergePAMs = False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualization of PAM-PAM synapses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RETRIEVE ALL PAM-PAM SYNAPSES\n",
    "\n",
    "## get PAM neurons \n",
    "neuron_criteria = NC(status='Traced', type=\"^PAM.*\",regex=True)\n",
    "\n",
    "## get all synapses, within and outside MB\n",
    "allPAMPAMpresynapses_criteria = SC(type='pre', primary_only=True)\n",
    "allPAMPAMpostsynapses_criteria = SC(type='post', primary_only=True)\n",
    "#all_PAM_PAM_presynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, allPAMPAMpresynapses_criteria,batch_size=10000)\n",
    "#all_PAM_PAM_postsynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, allPAMPAMpostsynapses_criteria,batch_size=10000)\n",
    "### loads cached version of above dataframes\n",
    "all_PAM_PAM_presynapses = loadPickle('all_PAM_PAM_presynapses')\n",
    "all_PAM_PAM_postsynapses = loadPickle('all_PAM_PAM_postsynapses')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import plotSynapseConfidence\n",
    "\n",
    "\n",
    "plotSynapseConfidence(all_PAM_PAM_presynapses, mode=\"pre\")\n",
    "plotSynapseConfidence(all_PAM_PAM_postsynapses, mode=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import filterSynapseConfidence\n",
    "\n",
    "PAMPAMpresynapses = filterSynapseConfidence(all_PAM_PAM_presynapses,mode=\"pre\")\n",
    "PAMPAMpostsynapses = filterSynapseConfidence(all_PAM_PAM_postsynapses,mode=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(rois)\n",
    "from utils import getAnatomicalOutlines, plotSynapseGroups, plotSynapseClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "#fetch 2d outlines of mushroombody (based on neuprint mesh)\n",
    "MB_outlines_xz = getAnatomicalOutlines(\"MB\")\n",
    "MB_outlines_xy = getAnatomicalOutlines(\"MB\",'xy')\n",
    "\n",
    "## fetch alpha lobe\n",
    "aL_outlines_xz = getAnatomicalOutlines(\"aL\")\n",
    "aL_outlines_xy = getAnatomicalOutlines(\"aL\",'xy')\n",
    "\"\"\"\n",
    "\n",
    "#fetch 2d outlines of mushroombody (based on neuprint mesh)\n",
    "MB_outlines_xz = loadPickle(\"MB_outlines_xz\")\n",
    "MB_outlines_xy = loadPickle(\"MB_outlines_xy\")\n",
    "\n",
    "## fetch alpha lobe\n",
    "aL_outlines_xz = loadPickle(\"aL_outlines_xz\")\n",
    "aL_outlines_xy = loadPickle(\"aL_outlines_xy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MB_outlines_xz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title = \"PAM-PAM synapses (red: pre, blue: post)\"\n",
    "plotSynapseGroups([PAMPAMpresynapses,PAMPAMpostsynapses],title,ROIoutlines=MB_outlines_xz,ROIname=\"MB outline\")\n",
    "plotSynapseGroups([PAMPAMpresynapses,PAMPAMpostsynapses],title, coordinates=[\"x\",\"y\"],ROIoutlines=MB_outlines_xy,ROIname=\"MB outline\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Printing ROIs with PAM-PAM presynapses\")\n",
    "print(sorted(filter(None, PAMPAMpresynapses[\"roi_pre\"].unique())))\n",
    "print(sorted(filter(None, PAMPAMpostsynapses[\"roi_pre\"].unique())))\n",
    "print(\"\\nPrinting ROIs with PAM-PAM postsynapses\")\n",
    "print(sorted(filter(None, PAMPAMpresynapses[\"roi_post\"].unique())))\n",
    "print(sorted(filter(None, PAMPAMpostsynapses[\"roi_post\"].unique())))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "noneRoiPresynapses = PAMPAMpresynapses[all_PAM_PAM_presynapses[\"roi_post\"].isnull()]\n",
    "noneRoiPostsynapses = PAMPAMpostsynapses[all_PAM_PAM_postsynapses[\"roi_post\"].isnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### analyzing MB and non-MB synapses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## get PAM neurons \n",
    "neuron_criteria = NC(status='Traced', type=\"^PAM.*\",regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## get synapses within MB\n",
    "MB_rois=[\"a'L(R)\",\"aL(R)\",\"b'L(R)\",\"bL(R)\",\"gL(R)\",\"CA(L)\",\"a'L(L)\",\"aL(L)\",\"b'L(L)\",\"bL(L)\",\"gL(L)\", \"CA(R)\", \"PED(R)\"]\n",
    "MBpresynapses_criteria = SC(type='pre', primary_only=True,rois=MB_rois)\n",
    "MBpostsynapses_criteria = SC(type='post', primary_only=True,rois=MB_rois)\n",
    "#MB_PAM_PAM_presynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, MBpresynapses_criteria)\n",
    "#MB_PAM_PAM_postsynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, MBpostsynapses_criteria)\n",
    "MB_PAM_PAM_presynapses = loadPickle(\"MB_PAM_PAM_presynapses\")\n",
    "MB_PAM_PAM_postsynapses = loadPickle(\"MB_PAM_PAM_postsynapses\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## get synapses outside MB\n",
    "### previously, these ROIs have been found to be the ROIs outside the MB that have PAM-PAM connections\n",
    "non_MB_rois=[\"CRE(L)\", \"CRE(R)\", \"EB\", \"LAL(R)\", \"SIP(L)\", \"SIP(R)\", \"SLP(R)\", \"SMP(L)\", \"SMP(R)\", \"LAL(L)\"]\n",
    "nonMBpresynapses_criteria = SC(type='pre', primary_only=True,rois=non_MB_rois)\n",
    "nonMBpostsynapses_criteria = SC(type='post', primary_only=True,rois=non_MB_rois)\n",
    "#nonMB_PAM_PAM_presynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, nonMBpresynapses_criteria)\n",
    "#nonMB_PAM_PAM_postsynapses = fetch_synapse_connections(neuron_criteria, neuron_criteria, nonMBpostsynapses_criteria)\n",
    "nonMB_PAM_PAM_presynapses = loadPickle(\"nonMB_PAM_PAM_presynapses\")\n",
    "nonMB_PAM_PAM_postsynapses = loadPickle(\"nonMB_PAM_PAM_postsynapses\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MBpresynapses = filterSynapseConfidence(MB_PAM_PAM_presynapses,mode=\"pre\")\n",
    "MBpostsynapses = filterSynapseConfidence(MB_PAM_PAM_postsynapses,mode=\"post\")\n",
    "nonMBpresynapses = filterSynapseConfidence(nonMB_PAM_PAM_presynapses,mode=\"pre\")\n",
    "nonMBpostsynapses = filterSynapseConfidence(nonMB_PAM_PAM_postsynapses,mode=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(MBpresynapses.index.size)\n",
    "print(MBpostsynapses.index.size)\n",
    "print(nonMBpresynapses.index.size)\n",
    "print(nonMBpostsynapses.index.size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filteredMBpre=filterSynapseConfidence(MBpresynapses,mode=\"pre\",percentile=90)\n",
    "filterednonMBpre=filterSynapseConfidence(nonMBpresynapses,mode=\"pre\",percentile=90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(filteredMBpre.index.size)\n",
    "print(filterednonMBpre.index.size)\n",
    "# todo investigate these non-MB-synapses coordinates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualization of PAM-PAM synapses inside/outside MB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the synapse positions in a 2D projection\n",
    "## blue synapses are MB synapses, red are outside the MB\n",
    "title = \"PAM-PAM synapses (red: MB, blue: non-MB, green: ROI 'None')\"\n",
    "\n",
    "plotSynapseGroups([MBpresynapses,nonMBpresynapses,noneRoiPresynapses],title,ROIoutlines=MB_outlines_xz,ROIname=\"MB outline\")\n",
    "plotSynapseGroups([MBpresynapses,nonMBpresynapses,noneRoiPresynapses],title, coordinates=[\"x\",\"y\"],ROIoutlines=MB_outlines_xy,ROIname=\"MB outline\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### analysis of synapse details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_PAM_PAM_presynapses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_neurons, roi_count = neu.fetch_neurons(NC(status='Traced',type=\"^.*\",regex=True)) \n",
    "all_neurons = loadPickle(\"all_neurons\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import collapseNeuronNames, classifySynapseConnectivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedPPpresynapses = neu.merge_neuron_properties(all_neurons,all_PAM_PAM_presynapses,properties=[\"type\", \"instance\"])\n",
    "mergedPPpostynapses = neu.merge_neuron_properties(all_neurons,all_PAM_PAM_postsynapses,properties=[\"type\", \"instance\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedPPpresynapses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import filterSynapseConfidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title = \"PAM-PAM Synapse Types\"\n",
    "mergedPPpresynapses = classifySynapseConnectivity(mergedPPpresynapses)\n",
    "mergedPPpostynapses = classifySynapseConnectivity(mergedPPpostynapses)\n",
    "plotSynapseClassification(mergedPPpresynapses,title=title,ROIoutlines=MB_outlines_xz,ROIname=\"MB outline\")\n",
    "plotSynapseClassification(mergedPPpresynapses, title=title, coordinates=[\"x\",\"y\"],ROIoutlines=MB_outlines_xy,ROIname=\"MB outline\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = neu.merge_neuron_properties(all_neurons,nonMBpostsynapses,properties=[\"type\", \"instance\"])\n",
    "classified = classifySynapseConnectivity(merged)\n",
    "filtered=classified[classified[\"type_post\"]==\"PAM01_a\"]\n",
    "plotSynapseClassification(filtered,title=title,ROIoutlines=MB_outlines_xz,ROIname=\"MB outline\")\n",
    "plotSynapseClassification(filtered, title=title, coordinates=[\"x\",\"y\"],ROIoutlines=MB_outlines_xy,ROIname=\"MB outline\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## fetch all synapses that involve PAM,\n",
    "## postsynapses are 1:1 identical to presynapses, both have n=150049 \n",
    "pam_criteria = NC(status='Traced', type=\"^PAM.*\",regex=True)\n",
    "all_criteria = NC(status='Traced', type=\"^.*\",regex=True)\n",
    "\n",
    "allPAMpresynapses_criteria = SC(type='pre', primary_only=True)\n",
    "#allPAMpostsynapses_criteria = SC(type='post', primary_only=True)\n",
    "#all_PAM_All_presynapses = fetch_synapse_connections(pam_criteria, all_criteria, allPAMpresynapses_criteria,batch_size=10000)\n",
    "#all_PAM_All_postsynapses = fetch_synapse_connections(all_criteria, pam_criteria, allPAMpostsynapses_criteria,batch_size=10000)\n",
    "all_PAM_All_presynapses = loadPickle(\"all_PAM_All_presynapses\")\n",
    "all_PAM_All_postsynapses = loadPickle(\"all_PAM_All_postsynapses\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAMpresynapses = filterSynapseConfidence(all_PAM_All_presynapses,mode=\"pre\",percentile=10) \n",
    "mergedPAMpresynapses = neu.merge_neuron_properties(all_neurons,PAMpresynapses,properties=[\"type\", \"instance\"])\n",
    "#mergedPAMpostynapses = neu.merge_neuron_properties(all_neurons,allPAMpostsynapses,properties=[\"type\", \"instance\"])\n",
    "\n",
    "mergedPAMpresynapses = classifySynapseConnectivity(mergedPAMpresynapses)\n",
    "#mergedPAMpostynapses = classify_connections(mergedPAMpostynapses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title=\"PAM Presynapses Classification\"\n",
    "decimated_connections = mergedPAMpresynapses.sample(n=10000)\n",
    "plotSynapseClassification(decimated_connections, title=title, coordinates=[\"x\",\"z\"],ROIoutlines=MB_outlines_xz,ROIname=\"MB\")\n",
    "plotSynapseClassification(decimated_connections, title=title, coordinates=[\"x\",\"y\"],ROIoutlines=MB_outlines_xy,ROIname=\"MB\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import importlib\n",
    "importlib.reload(utils)\n",
    "from utils import plotPAMTypePresynapses\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotPAMTypePresynapses(mergedPAMpresynapses,pamType=\"PAM01\",roi_outlines=[MB_outlines_xz,MB_outlines_xy],roiName=\"MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neurotransmitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neurotransmitters = pd.read_feather(\"pickles/Hemibrain Neurotransmitters.feather\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neurotransmitters.sort_values(\"nts_8.dopamine\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import getROIs\n",
    "\n",
    "MBroisRegex = getROIs(\"MB\",\"regexOr\")\n",
    "nonMBroisRegex = getROIs(\"nonMB\",\"regexOr\")\n",
    "## regex search term for all ROIs that we know to contain PAM neurons\n",
    "allROIsRegex = MBroisRegex + \"|\" + nonMBroisRegex\n",
    "allROIsRegex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neurotransmitters_filtered = neurotransmitters[neurotransmitters['roi'].str.contains(allROIsRegex,regex=True)]\n",
    "neurotransmitters_filtered\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAMPAMpresynapses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming PAM_PAM_connections is already defined somewhere in the notebook\n",
    "merged_df = pd.merge(neurotransmitters, PAMPAMpostsynapses, left_on=['sv', 'body'], right_on=['bodyId_pre', 'bodyId_post'])\n",
    "merged_df\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DANModulation",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}