gold_relevance_scores.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scripts.utils import read_gold_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path_name = \"data-release\" # set data release path name\n",
    "\n",
    "data = read_gold_data(data_path_name)\n",
    "corpus = data[\"corpus\"]\n",
    "baseline_queries_train = data[\"baseline\"][\"train\"]\n",
    "baseline_queries_dev = data[\"baseline\"][\"dev\"]\n",
    "perspective_queries_train = data[\"perspective\"][\"train\"]\n",
    "perspective_queries_dev = data[\"perspective\"][\"dev\"]\n",
    "\n",
    "# uncomment when working with test set\n",
    "# baseline_queries_test = data[\"baseline\"][\"test\"]\n",
    "# perspective_queries_test = data[\"perspective\"][\"test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_relevance_scores = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "queries = baseline_queries_dev # set correct set of queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "argument_ids = list(corpus[\"argument_id\"].values)\n",
    "relevant_candidates_lists = list(queries[\"relevant_candidates\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def row_gold_scores(argument_ids, candidates_list):\n",
    "    scores = [1 if i in candidates_list else 0 for i in argument_ids]\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_scores = [row_gold_scores(argument_ids, candidates_list) for candidates_list in relevant_candidates_lists]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_scores_dict = []\n",
    "for i in range(len(gold_scores)):\n",
    "    gold_scores_dict.append({\n",
    "        'query_id': queries.iloc[i][\"query_id\"],\n",
    "        'gold_scores': gold_scores[i]\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_scores_df = pd.DataFrame(gold_scores_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_path = \"example_path/example_gold_scores.jsonl\" # set path, where gold relevance scores are saved in jsonl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_scores_df.to_json(output_path, orient=\"records\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}