etk_date_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Custom Spacy Rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc = {\"doc_id\": \"de60d1fd856b8c49568eba4f5026c58c8e7c09f4\", \"measurement\": {\"date\": \"2013-01-28\", \"type\": \"Measurement\", \"number\": 113.92, \"timeseries\": \"77e7e461cf45417a99c3d0f5eb17ab3533093495\"}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "etk_config = {\n",
    "    \"document_id\": \"doc_id\",\n",
    "    \"resources\": {\n",
    "    \"stop_word_dictionaries\": {},\n",
    "    \"landmark\": [],\n",
    "    \"dictionaries\": {},\n",
    "    \"spacy_field_rules\": {\n",
    "      \"measurement_date_rules\": \"/Users/amandeep/Github/etk/etk/resources/measurement_date_rules.json\"\n",
    "    }\n",
    "  },\n",
    "  \"content_extraction\": {\n",
    "    \"json_content\": [\n",
    "      {\n",
    "        \"input_path\": \"measurement.type\", \n",
    "        \"segment_name\": \"measurement__type\"\n",
    "      }, \n",
    "      {\n",
    "        \"input_path\": \"measurement.date\", \n",
    "        \"segment_name\": \"measurement__date\"\n",
    "      }, \n",
    "      {\n",
    "        \"input_path\": \"measurement.number\", \n",
    "        \"segment_name\": \"measurement__number\"\n",
    "      }, \n",
    "      {\n",
    "        \"input_path\": \"measurement.timeseries\", \n",
    "        \"segment_name\": \"measurement__timeseries\"\n",
    "      }\n",
    "    ]\n",
    "  }, \n",
    "  \"data_extraction\": [\n",
    "    {\n",
    "      \"fields\": {\n",
    "        \"type\": {\n",
    "          \"extractors\": {\n",
    "            \"extract_as_is\": {}\n",
    "          }\n",
    "        }\n",
    "      }, \n",
    "      \"input_path\": \"content_extraction.measurement__type[*]\"\n",
    "    }, \n",
    "    {\n",
    "      \"fields\": {\n",
    "        \"event_date\": {\n",
    "          \"extractors\": {\n",
    "            \"extract_as_is\": {\n",
    "              \"config\": {\n",
    "                \"ignore_past_years\": 100, \n",
    "                \"post_filter\": \"parse_date_generic\"\n",
    "              }\n",
    "            },\n",
    "            \"extract_using_custom_spacy\": {\n",
    "                \"config\": {\n",
    "                  \"spacy_field_rules\": \"measurement_date_rules\",\n",
    "                  \"post_filter\": \"parse_date_generic\"\n",
    "                }\n",
    "              }\n",
    "          }\n",
    "        }\n",
    "      }, \n",
    "      \"input_path\": \"content_extraction.measurement__date[*]\"\n",
    "    }, \n",
    "    {\n",
    "      \"fields\": {\n",
    "        \"value\": {\n",
    "          \"extractors\": {\n",
    "            \"extract_as_is\": {}\n",
    "          }\n",
    "        }\n",
    "      }, \n",
    "      \"input_path\": \"content_extraction.measurement__number[*]\"\n",
    "    }, \n",
    "    {\n",
    "      \"fields\": {\n",
    "        \"measure\": {\n",
    "          \"extractors\": {\n",
    "            \"extract_as_is\": {}\n",
    "          }\n",
    "        }\n",
    "      }, \n",
    "      \"input_path\": \"content_extraction.measurement__timeseries[*]\"\n",
    "    }\n",
    "  ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  {\n",
      "    \"confidence\": 1.0, \n",
      "    \"provenance\": [\n",
      "      {\n",
      "        \"source\": {\n",
      "          \"segment\": \"content_extraction.measurement__date.[0]\", \n",
      "          \"document_id\": \"de60d1fd856b8c49568eba4f5026c58c8e7c09f4\"\n",
      "        }, \n",
      "        \"confidence\": {\n",
      "          \"extraction\": 1.0\n",
      "        }, \n",
      "        \"method\": \"extract_as_is\", \n",
      "        \"extracted_value\": \"2013-01-28T00:00:00\"\n",
      "      }, \n",
      "      {\n",
      "        \"source\": {\n",
      "          \"segment\": \"other_segment\", \n",
      "          \"context\": {\n",
      "            \"end\": 5, \n",
      "            \"text\": \" <etk 'attribute' = 'event_date'>2013 - 01 - 28</etk>  \", \n",
      "            \"start\": 0, \n",
      "            \"input\": \"tokens\", \n",
      "            \"identifier\": \"rule_4\", \n",
      "            \"rule_id\": 3\n",
      "          }, \n",
      "          \"document_id\": \"de60d1fd856b8c49568eba4f5026c58c8e7c09f4\"\n",
      "        }, \n",
      "        \"confidence\": {\n",
      "          \"extraction\": 1.0\n",
      "        }, \n",
      "        \"method\": \"extract_using_custom_spacy\", \n",
      "        \"extracted_value\": \"2013-01-28T00:00:00\"\n",
      "      }\n",
      "    ], \n",
      "    \"key\": \"2013-01-28t00:00:00\", \n",
      "    \"value\": \"2013-01-28T00:00:00\"\n",
      "  }\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from etk.core import Core\n",
    "\n",
    "c = Core(extraction_config=etk_config)\n",
    "r = c.process(doc)\n",
    "print json.dumps(r[\"knowledge_graph\"][\"event_date\"], indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:etk_env]",
   "language": "python",
   "name": "conda-env-etk_env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}