project_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "naval-relative",
   "metadata": {},
   "source": [
    "cat project_data.zip | ssh -T vshishkov-243998@gateway.st \"kubectl exec --stdin pod/jupyter-spark-7d48566c67-58jql -- hdfs dfs -put -f - /tmp/vshishkov-243998/project_data.zip\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "characteristic-judges",
   "metadata": {},
   "source": [
    "HDFS -> Local -> Extracting -> HDFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "saving-collapse",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !if [ ! -f \"/home/jovyan/work/project_data.zip\" ]; then hdfs dfs -copyToLocal /tmp/vshishkov-243998/project_data.zip ~/work/project_data.zip; fi; \n",
    "# !if [ ! -d \"/home/jovyan/work/project_data\" ]; then unzip -qq ~/work/project_data.zip -d ~/work; fi;\n",
    "# !hdfs dfs -rm -r /tmp/vshishkov-243998/project_data\n",
    "# !hdfs dfs -put -f ~/work/project_data /tmp/vshishkov-243998/project_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pleased-works",
   "metadata": {},
   "source": [
    "pip install \\\n",
    "    --user \\\n",
    "    --trusted-host pypi-service.pypi \\\n",
    "    --index-url http://pypi-service.pypi:8080/ \\\n",
    "    sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aboriginal-girlfriend",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "satisfactory-grass",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import socket\n",
    "import datetime\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from pyspark import SparkConf, SparkContext, SQLContext\n",
    "from pyspark.sql import SparkSession, Window\n",
    "from pyspark.sql.functions import udf, length, when, col\n",
    "import pyspark.sql.types as ps_types\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "from pyspark.ml.feature import VectorAssembler, StandardScaler\n",
    "from pyspark.ml.regression import LinearRegression, RandomForestRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "peaceful-crazy",
   "metadata": {},
   "source": [
    "Base dirs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "turned-limitation",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = Path(\"/home/jovyan/work/project_data\")\n",
    "currencies_dir = data_dir / \"currencies\"\n",
    "gtrends_dir = data_dir / \"google trends\"\n",
    "reddit_dir = data_dir / \"reddit\"\n",
    "twitter_dir = data_dir / \"twitter/combined_all_coins_tweet_data\"\n",
    "\n",
    "hdfs_data_dir = Path(\"hdfs://tmp/vshishkov-243998/project_data\")\n",
    "hdfs_currencies_dir = hdfs_data_dir / \"currencies\"\n",
    "hdfs_gtrends_dir = hdfs_data_dir / \"google trends\"\n",
    "hdfs_reddit_dir = hdfs_data_dir / \"reddit\"\n",
    "hdfs_twitter_dir = hdfs_data_dir / \"twitter/combined_all_coins_tweet_data\"\n",
    "hdfs_models_dir = hdfs_data_dir / \"models\"\n",
    "hdfs_preds_dir = hdfs_data_dir / \"preds\"\n",
    "hdfs_scalers_dir = hdfs_data_dir / \"scalers\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "successful-volume",
   "metadata": {},
   "source": [
    "Spark session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "opposed-atlas",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOCAL_IP = socket.gethostbyname(socket.gethostname())\n",
    "spark = (SparkSession\n",
    "    .builder\n",
    "    .master('k8s://https://10.32.7.103:6443')\n",
    "    .config('spark.driver.host', LOCAL_IP)\n",
    "    .config('spark.driver.bindAddress', '0.0.0.0')\n",
    "    .config('spark.executor.instances', '2')\n",
    "    .config('spark.executor.cores', '2')\n",
    "    .config('spark.cores.max', '4')\n",
    "    .config('spark.executor.memory', '4g')\n",
    "    .config('spark.sql.execution.arrow.enabled', 'true')\n",
    "    .config('spark.kubernetes.namespace', 'vshishkov-243998')\n",
    "    .config('spark.kubernetes.container.image', 'node03.st:5000/spark-executor:vshishkov-243998')\n",
    "    .config('spark.kubernetes.container.image.pullPolicy', 'Always')\n",
    "    .config('spark.kubernetes.executor.deleteOnTermination', 'false')\n",
    "    .getOrCreate()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "simplified-associate",
   "metadata": {},
   "source": [
    "Reading metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "given-romania",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# all metadata in format { lowered ticker : hdfs path to file }\n",
    "\n",
    "# currencies data\n",
    "coins_meta = pd.read_csv(currencies_dir / \"coins.csv\")\n",
    "coins_meta = {x.lower(): (hdfs_currencies_dir / f\"{x.upper()}.csv\").as_posix() for x in coins_meta.Coin}\n",
    "\n",
    "# google trends data\n",
    "gtrends_data = spark.read.option(\"multiline\", True).option('escape', \"\\\"\").csv((hdfs_gtrends_dir / \"GT_adj.csv\").as_posix(), header=True, inferSchema=True)\\\n",
    "    .withColumnRenamed(\"_c0\", \"gtrends_date\")\\\n",
    "    .withColumn(\"gtrends_date\", F.to_timestamp(col(\"gtrends_date\"), \"yyyy-MM-dd\"))\n",
    "gtrends_meta = [x.split(\"_\")[0].lower() + \"-usd\" for x in gtrends_data.columns[1:]]\n",
    "\n",
    "# reddit data\n",
    "reddit_meta = pd.read_csv(reddit_dir / \"meta_info.csv\")\n",
    "reddit_meta = {row.ticker.lower(): (hdfs_reddit_dir / f\"{row.subreddit}_features.csv\").as_posix() for _, row in reddit_meta.iterrows()}\n",
    "\n",
    "# twitter data\n",
    "twitter_meta = {x.stem.lstrip(\"tweets_\") + \"-usd\": Path(*hdfs_data_dir.parts, *[x for x in (twitter_dir / x.stem).glob(\"*\") if \"feat\" in x.stem][0].parts[- 4:]).as_posix() for x in twitter_dir.iterdir()}\n",
    "\n",
    "# intersection\n",
    "coins_all = set(coins_meta.keys()) & set(reddit_meta.keys()) & set(twitter_meta.keys() & set(gtrends_meta))\n",
    "len(coins_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "lovely-dependence",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_lags(df, date_col, lags_num, cols_to_lag):\n",
    "    w = Window().partitionBy().orderBy(col(date_col))\n",
    "    return df.select([date_col] + [F.lag(col_to_lag, lag).over(w).alias(f\"{col_to_lag}_{lag}\")\\\n",
    "                     for lag in range(1, lags_num + 1) for col_to_lag in cols_to_lag]).na.drop()\n",
    "\n",
    "get_item0_udf = F.udf(lambda x : float(x[0]), ps_types.DoubleType())\n",
    "\n",
    "def create_train_test(df, date_col, target_col, lags_num, test_size, scale=False):\n",
    "    # creating lags\n",
    "    df = df.join(create_lags(df, date_col=date_col, lags_num=lags_num, \n",
    "                             cols_to_lag=[x for x in coin_data.columns if x != date_col]),\n",
    "               on=date_col, how=\"inner\").sort(\"Date\")\n",
    "    \n",
    "    # splitting\n",
    "    train_len = round((1 - test_size) * df.count())\n",
    "    train_df = df.limit(train_len)\n",
    "    test_df = df.subtract(train_df)\n",
    "    \n",
    "    # assembling\n",
    "    va_features = VectorAssembler(inputCols=[x for x in df.columns if x != date_col and x != target_col], outputCol=\"features\")\n",
    "    va_target = VectorAssembler(inputCols=[target_col], outputCol=\"target\")\n",
    "    va_transform = lambda x: va_target.transform(va_features.transform(x)).select(\"features\", \"target\", \"Date\")\n",
    "    train, test = map(va_transform, (train_df, test_df))\n",
    "    \n",
    "    # scaling\n",
    "    if scale:\n",
    "        features_scaler = StandardScaler(inputCol=\"features\", outputCol=\"scaled_features\",\n",
    "                            withStd=True, withMean=True).fit(train)\n",
    "        target_scaler = StandardScaler(inputCol=\"target\", outputCol=\"scaled_target\",\n",
    "                            withStd=True, withMean=True).fit(train)\n",
    "        scale = lambda x: target_scaler.transform(features_scaler.transform(x))\\\n",
    "            .select(col(\"scaled_features\").alias(\"features\"), get_item0_udf(col(\"scaled_target\")).alias(\"target\"), \"Date\")\n",
    "#         mean = train_df.agg(F.mean(col(target_col)).alias(\"mean\")).collect()[0]['mean']\n",
    "#         std = train_df.agg(F.stddev(col(target_col)).alias(\"std\")).collect()[0]['std']\n",
    "#         scale = lambda x: features_scaler.transform(x)\\\n",
    "#             .select(col(\"scaled_features\").alias(\"features\"), ((get_item0_udf(col(\"target\")) - mean) / std).alias(\"target\"))\n",
    "    else:\n",
    "        features_scaler = target_scaler = None\n",
    "        features_scaler = StandardScaler(inputCol=\"features\", outputCol=\"scaled_features\",\n",
    "                            withStd=True, withMean=True).fit(train)\n",
    "        scale = lambda x: features_scaler.transform(x).select(col(\"scaled_features\").alias(\"features\"),\n",
    "                    get_item0_udf(col(\"target\")).alias(\"target\"), \"Date\")\n",
    "    train, test = map(scale, (train, test))\n",
    "    \n",
    "    return train, test, features_scaler, target_scaler\n",
    "\n",
    "def get_mae(preds):\n",
    "    return preds.select(F.sum(F.abs(col(\"prediction\") - col(\"target\"))).alias(\"MAE\")).collect()[0].MAE\n",
    "\n",
    "def eval_model(model, train, test, metric_func):\n",
    "    model = model.fit(train)\n",
    "    preds = model.transform(test)\n",
    "    metric = metric_func(preds)\n",
    "    return model, metric, preds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hairy-proof",
   "metadata": {},
   "source": [
    "Training / evaluation routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "specific-courtesy",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 5/18 [25:38<1:01:46, 285.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o4283.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 80 in stage 1027.0 failed 4 times, most recent failure: Lost task 80.3 in stage 1027.0 (TID 20731) (10.128.128.52 executor 8): org.apache.spark.SparkException: EOFException occurred while reading the port number from pyspark.daemon's stdout\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.startDaemon(PythonWorkerFactory.scala:217)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:132)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n",
      "\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: EOFException occurred while reading the port number from pyspark.daemon's stdout\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.startDaemon(PythonWorkerFactory.scala:217)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:132)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 15/18 [1:11:01<11:16, 225.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o12139.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 50 in stage 2994.0 failed 4 times, most recent failure: Lost task 50.3 in stage 2994.0 (TID 56249) (10.128.128.61 executor 21): java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createSocket$1(PythonWorkerFactory.scala:120)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.liftedTree1$1(PythonWorkerFactory.scala:136)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:135)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createSocket$1(PythonWorkerFactory.scala:120)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.liftedTree1$1(PythonWorkerFactory.scala:136)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:135)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 16/18 [1:17:34<09:11, 275.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o13362.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 3269.0 failed 4 times, most recent failure: Lost task 2.3 in stage 3269.0 (TID 62017) (10.128.117.215 executor 22): org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage30.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage30.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18/18 [1:28:55<00:00, 296.42s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>coin</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>bnb-usd</td>\n",
       "      <td>58.141663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>bnb-usd</td>\n",
       "      <td>115.422201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>xmr-usd</td>\n",
       "      <td>15.386517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>xmr-usd</td>\n",
       "      <td>30.116431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>70.721581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>3669.071687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>xrp-usd</td>\n",
       "      <td>13.524856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>xrp-usd</td>\n",
       "      <td>37.402920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>dot-usd</td>\n",
       "      <td>6.954707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>dot-usd</td>\n",
       "      <td>11.225788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>link-usd</td>\n",
       "      <td>13.667975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>link-usd</td>\n",
       "      <td>39.958081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>aave-usd</td>\n",
       "      <td>3.389094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>aave-usd</td>\n",
       "      <td>12.178770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>uni1-usd</td>\n",
       "      <td>2.162287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>uni1-usd</td>\n",
       "      <td>9.282443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>vet-usd</td>\n",
       "      <td>15.749180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>vet-usd</td>\n",
       "      <td>45.097707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>fil-usd</td>\n",
       "      <td>11.821681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>fil-usd</td>\n",
       "      <td>42.683028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>zil-usd</td>\n",
       "      <td>107.979514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>zil-usd</td>\n",
       "      <td>25.974612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>lrc-usd</td>\n",
       "      <td>51.500280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>lrc-usd</td>\n",
       "      <td>344.477636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>doge-usd</td>\n",
       "      <td>16.111171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>doge-usd</td>\n",
       "      <td>116.331069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>rep-usd</td>\n",
       "      <td>12.277490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>rep-usd</td>\n",
       "      <td>14.646858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>icp-usd</td>\n",
       "      <td>0.605456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>icp-usd</td>\n",
       "      <td>7.753270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>ltc-usd</td>\n",
       "      <td>41.574360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>ltc-usd</td>\n",
       "      <td>199.724065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>ada-usd</td>\n",
       "      <td>53.114603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>ada-usd</td>\n",
       "      <td>190.433189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>axs-usd</td>\n",
       "      <td>2.493183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>axs-usd</td>\n",
       "      <td>10.977394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model      coin       metric\n",
       "0        LinearRegression   bnb-usd    58.141663\n",
       "1   RandomForestRegressor   bnb-usd   115.422201\n",
       "2        LinearRegression   xmr-usd    15.386517\n",
       "3   RandomForestRegressor   xmr-usd    30.116431\n",
       "4        LinearRegression   btc-usd    70.721581\n",
       "5   RandomForestRegressor   btc-usd  3669.071687\n",
       "6        LinearRegression   xrp-usd    13.524856\n",
       "7   RandomForestRegressor   xrp-usd    37.402920\n",
       "8        LinearRegression   dot-usd     6.954707\n",
       "9   RandomForestRegressor   dot-usd    11.225788\n",
       "10       LinearRegression  link-usd    13.667975\n",
       "11  RandomForestRegressor  link-usd    39.958081\n",
       "12       LinearRegression  aave-usd     3.389094\n",
       "13  RandomForestRegressor  aave-usd    12.178770\n",
       "14       LinearRegression  uni1-usd     2.162287\n",
       "15  RandomForestRegressor  uni1-usd     9.282443\n",
       "16       LinearRegression   vet-usd    15.749180\n",
       "17  RandomForestRegressor   vet-usd    45.097707\n",
       "18       LinearRegression   fil-usd    11.821681\n",
       "19  RandomForestRegressor   fil-usd    42.683028\n",
       "20       LinearRegression   zil-usd   107.979514\n",
       "21  RandomForestRegressor   zil-usd    25.974612\n",
       "22       LinearRegression   lrc-usd    51.500280\n",
       "23  RandomForestRegressor   lrc-usd   344.477636\n",
       "24       LinearRegression  doge-usd    16.111171\n",
       "25  RandomForestRegressor  doge-usd   116.331069\n",
       "26       LinearRegression   rep-usd    12.277490\n",
       "27  RandomForestRegressor   rep-usd    14.646858\n",
       "28       LinearRegression   icp-usd     0.605456\n",
       "29  RandomForestRegressor   icp-usd     7.753270\n",
       "30       LinearRegression   ltc-usd    41.574360\n",
       "31  RandomForestRegressor   ltc-usd   199.724065\n",
       "32       LinearRegression   ada-usd    53.114603\n",
       "33  RandomForestRegressor   ada-usd   190.433189\n",
       "34       LinearRegression   axs-usd     2.493183\n",
       "35  RandomForestRegressor   axs-usd    10.977394"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_part = 0.2\n",
    "models = [LinearRegression, RandomForestRegressor]\n",
    "models_params = [{\"featuresCol\": \"features\", \"labelCol\": \"target\"},\n",
    "                {\"featuresCol\": \"features\", \"labelCol\": \"target\", \"numTrees\": 10, \"maxDepth\": 5, \"seed\": 42}]\n",
    "results = {\"model\": [], \"coin\": [], \"metric\": []}\n",
    "\n",
    "for coin in tqdm(coins_all):\n",
    "    flag = False\n",
    "    while not flag:\n",
    "        try:\n",
    "            #  reading all the data\n",
    "            coin_data = spark.read.option(\"mode\", \"DROPMALFORMED\").csv(coins_meta[coin], header=True, inferSchema=True).drop(\"Dividends\", \"Stock Splits\")\n",
    "            coin_data = coin_data.withColumn(\"Date\", F.to_timestamp(coin_data.Date, \"yyyy-MM-dd\"))\n",
    "\n",
    "            coin_gtrends_data = gtrends_data.select(\"gtrends_date\", f\"{coin[: - 4].upper()}_adj_svi\")\n",
    "\n",
    "            reddit_data = spark.read.option(\"mode\", \"DROPMALFORMED\").option(\"multiline\", True).option('escape', \"\\\"\").csv(reddit_meta[coin], header=True, inferSchema=True)\n",
    "            reddit_data = reddit_data.withColumn(\"date\", F.to_timestamp(reddit_data.date, \"yyyy-MM-dd\"))\\\n",
    "                            .withColumnRenamed(\"date\", \"reddit_date\")\n",
    "\n",
    "            twitter_data = spark.read.option(\"mode\", \"DROPMALFORMED\").csv(twitter_meta[coin], header=True, inferSchema=True).drop(\"_c0\", \"Unnamed: 0\")\n",
    "            twitter_data = twitter_data.withColumn(\"date\", F.to_timestamp(twitter_data.date, \"yyyy-MM-dd\"))\\\n",
    "                            .withColumnRenamed(\"date\", \"twitter_date\")\n",
    "\n",
    "            #  merging data\n",
    "            tr_data = twitter_data.join(reddit_data, twitter_data.twitter_date == reddit_data.reddit_date, \"inner\")\n",
    "            ctr_data = coin_data.join(tr_data, coin_data.Date == tr_data.reddit_date, \"left\")\n",
    "            full_data = ctr_data.join(coin_gtrends_data, ctr_data.Date == coin_gtrends_data.gtrends_date, \"left\")\n",
    "            full_data = full_data.drop(\"reddit_date\", \"twitter_date\", \"gtrends_date\").fillna(0)\n",
    "\n",
    "            #  transformation to train / test\n",
    "            train, test, features_scaler, target_scaler = create_train_test(full_data, \"Date\", \"Close\", 1, 0.2, scale=True)\n",
    "            target_scaler.write().overwrite().save((hdfs_scalers_dir / coin).as_posix())\n",
    "            train.cache()\n",
    "            test.cache()\n",
    "\n",
    "            #  evaluation of the models\n",
    "            for model, model_params in zip(models, models_params):\n",
    "                model, metric, preds = eval_model(model(**model_params), train, test, get_mae)\n",
    "\n",
    "                model_type = model.uid.split('_')[0]\n",
    "                results[\"model\"].append(model_type)\n",
    "                results[\"coin\"].append(coin)\n",
    "                results[\"metric\"].append(metric)\n",
    "                model.write().overwrite().save((hdfs_models_dir / f\"{model_type}_{coin}\").as_posix())\n",
    "                preds.select(\"target\", \"prediction\").write.mode('overwrite').parquet((hdfs_preds_dir / f\"{model_type}_{coin}.parquet\").as_posix())\n",
    "\n",
    "            train.unpersist()\n",
    "            test.unpersist()\n",
    "            flag = True\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    \n",
    "results = pd.DataFrame(results)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "human-syria",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-------------------+-------------------+-------------------+\n",
      "|            features|             target|               Date|         prediction|\n",
      "+--------------------+-------------------+-------------------+-------------------+\n",
      "|[0.43962523970086...|0.33852174470613594|2022-02-11 00:00:00| 0.4079761059700112|\n",
      "|[0.33861415602650...| 0.3482065857091885|2022-02-12 00:00:00|0.35531184910968017|\n",
      "|[0.35011420615242...|  0.334398617204938|2022-02-13 00:00:00| 0.3416082644659399|\n",
      "|[0.33487845291639...|0.30178901403802894|2022-02-14 00:00:00| 0.3416082644659399|\n",
      "|[0.30388360189197...| 0.4342712315936625|2022-02-15 00:00:00| 0.3416082644659399|\n",
      "|[0.43619183827641...| 0.4402479748334851|2022-02-16 00:00:00| 0.3416082644659399|\n",
      "|[0.44151031844018...|0.28904807360203805|2022-02-17 00:00:00| 0.3753284671858196|\n",
      "|[0.29187727707208...| 0.2773517165520826|2022-02-18 00:00:00| 0.3416082644659399|\n",
      "|[0.27913065530612...| 0.2606887844689299|2022-02-19 00:00:00| 0.3416082644659399|\n",
      "|[0.26265894992719...| 0.2081523199038619|2022-02-20 00:00:00| 0.3158841525330616|\n",
      "|[0.20930628157144...|0.10630835363642507|2022-02-21 00:00:00| 0.2750180305702697|\n",
      "|[0.10814731156634...|0.18043904385245232|2022-02-22 00:00:00|0.15680415837906725|\n",
      "|[0.18273370381390...| 0.1366514376650575|2022-02-23 00:00:00|0.14987692944274042|\n",
      "|[0.13826412683343...|0.13439610149125736|2022-02-24 00:00:00|0.17788174893007444|\n",
      "|[0.13669692504226...|0.18220815733133516|2022-02-25 00:00:00| 0.1540867350440522|\n",
      "|[0.18443770046101...|0.16668958902852637|2022-02-26 00:00:00| 0.2108610572786799|\n",
      "|[0.16884749200885...| 0.1194974789374588|2022-02-27 00:00:00|0.17788174893007444|\n",
      "|[0.12142718941534...|0.25096834698740267|2022-02-28 00:00:00|0.16625093088189635|\n",
      "|[0.25307481441383...|0.30678646035344326|2022-03-01 00:00:00|0.29579922641039413|\n",
      "|[0.30876951453832...|0.27416858809308314|2022-03-02 00:00:00|0.35531184910968017|\n",
      "+--------------------+-------------------+-------------------+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "preds.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "retired-adams",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LinearRegression</th>\n",
       "      <td>27.620866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestRegressor</th>\n",
       "      <td>273.486508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           metric\n",
       "model                            \n",
       "LinearRegression        27.620866\n",
       "RandomForestRegressor  273.486508"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"model\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "individual-request",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.createDataFrame(results).write.mode('overwrite').csv((hdfs_data_dir / \"results.csv\").as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "grave-japan",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "aerial-rabbit",
   "metadata": {},
   "source": [
    "Without media"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "adult-classroom",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 1/18 [00:13<03:51, 13.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o15804.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 4 in stage 3770.0 failed 4 times, most recent failure: Lost task 4.3 in stage 3770.0 (TID 70175) (10.128.117.219 executor 26): org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 4/18 [03:33<10:29, 45.00s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o17859.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 3957.0 failed 4 times, most recent failure: Lost task 0.3 in stage 3957.0 (TID 71538) (10.128.128.59 executor 25): org.apache.spark.SparkException: EOFException occurred while reading the port number from pyspark.daemon's stdout\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.startDaemon(PythonWorkerFactory.scala:217)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:132)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: EOFException occurred while reading the port number from pyspark.daemon's stdout\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.startDaemon(PythonWorkerFactory.scala:217)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.createThroughDaemon(PythonWorkerFactory.scala:132)\n",
      "\tat org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:105)\n",
      "\tat org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)\n",
      "\tat org.apache.spark.sql.execution.python.BatchEvalPythonExec.evaluate(BatchEvalPythonExec.scala:70)\n",
      "\tat org.apache.spark.sql.execution.python.EvalPythonExec.$anonfun$doExecute$2(EvalPythonExec.scala:130)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:863)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:386)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 8/18 [06:36<05:27, 32.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o20462.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 185 in stage 4197.0 failed 4 times, most recent failure: Lost task 185.3 in stage 4197.0 (TID 73041) (10.128.117.209 executor 27): org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "An error occurred while calling o20870.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 4225.0 failed 4 times, most recent failure: Lost task 1.3 in stage 4225.0 (TID 73081) (10.128.117.209 executor 27): org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 16/18 [12:06<00:43, 21.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "An error occurred while calling o25670.collectToPython.\n",
      ": org.apache.spark.SparkException: Job aborted due to stage failure: Task 19 in stage 4677.0 failed 4 times, most recent failure: Lost task 19.3 in stage 4677.0 (TID 76554) (10.128.117.218 executor 29): org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n",
      "Driver stacktrace:\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2201)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)\n",
      "\tat scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)\n",
      "\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1078)\n",
      "\tat scala.Option.foreach(Option.scala:407)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1078)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)\n",
      "\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)\n",
      "\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n",
      "\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)\n",
      "\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n",
      "\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n",
      "\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:414)\n",
      "\tat org.apache.spark.rdd.RDD.collect(RDD.scala:1029)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:390)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$collectToPython$1(Dataset.scala:3519)\n",
      "\tat org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)\n",
      "\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772)\n",
      "\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)\n",
      "\tat org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685)\n",
      "\tat org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:3516)\n",
      "\tat jdk.internal.reflect.GeneratedMethodAccessor480.invoke(Unknown Source)\n",
      "\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n",
      "\tat java.base/java.lang.reflect.Method.invoke(Method.java:566)\n",
      "\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n",
      "\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n",
      "\tat py4j.Gateway.invoke(Gateway.java:282)\n",
      "\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n",
      "\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n",
      "\tat py4j.GatewayConnection.run(GatewayConnection.java:238)\n",
      "\tat java.base/java.lang.Thread.run(Thread.java:834)\n",
      "Caused by: org.apache.spark.SparkException: Python worker exited unexpectedly (crashed)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:550)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator$$anonfun$1.applyOrElse(PythonRunner.scala:539)\n",
      "\tat scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:38)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:89)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:67)\n",
      "\tat org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:470)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage10.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer$$anon$1.hasNext(InMemoryRelation.scala:118)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221)\n",
      "\tat org.apache.spark.storage.memory.MemoryStore.putIteratorAsValues(MemoryStore.scala:299)\n",
      "\tat org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1423)\n",
      "\tat org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1350)\n",
      "\tat org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1414)\n",
      "\tat org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1237)\n",
      "\tat org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:384)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:335)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)\n",
      "\tat org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n",
      "\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n",
      "\t... 1 more\n",
      "Caused by: java.io.EOFException\n",
      "\tat java.base/java.io.DataInputStream.readInt(DataInputStream.java:397)\n",
      "\tat org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:74)\n",
      "\t... 44 more\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18/18 [12:49<00:00, 42.74s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>coin</th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>bnb-usd</td>\n",
       "      <td>17.002522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>bnb-usd</td>\n",
       "      <td>140.459650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>xmr-usd</td>\n",
       "      <td>13.235198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>xmr-usd</td>\n",
       "      <td>27.408966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>69.574715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>3606.136179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>xrp-usd</td>\n",
       "      <td>12.317904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>xrp-usd</td>\n",
       "      <td>32.442800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>dot-usd</td>\n",
       "      <td>4.093729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>dot-usd</td>\n",
       "      <td>8.639492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>link-usd</td>\n",
       "      <td>13.634672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>link-usd</td>\n",
       "      <td>37.788630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>aave-usd</td>\n",
       "      <td>3.579303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>aave-usd</td>\n",
       "      <td>9.341335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>uni1-usd</td>\n",
       "      <td>2.036294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>uni1-usd</td>\n",
       "      <td>9.080294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>vet-usd</td>\n",
       "      <td>14.310090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>vet-usd</td>\n",
       "      <td>38.027085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>fil-usd</td>\n",
       "      <td>8.626917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>fil-usd</td>\n",
       "      <td>32.508731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>zil-usd</td>\n",
       "      <td>132.322583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>zil-usd</td>\n",
       "      <td>26.767661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>lrc-usd</td>\n",
       "      <td>41.360237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>lrc-usd</td>\n",
       "      <td>400.403597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>doge-usd</td>\n",
       "      <td>14.933210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>doge-usd</td>\n",
       "      <td>182.308127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>rep-usd</td>\n",
       "      <td>8.625588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>rep-usd</td>\n",
       "      <td>12.635247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>icp-usd</td>\n",
       "      <td>0.707277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>icp-usd</td>\n",
       "      <td>7.706200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>ltc-usd</td>\n",
       "      <td>34.609208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>ltc-usd</td>\n",
       "      <td>225.638346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>ada-usd</td>\n",
       "      <td>20.065167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>ada-usd</td>\n",
       "      <td>180.101381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>axs-usd</td>\n",
       "      <td>2.491264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>axs-usd</td>\n",
       "      <td>8.714931</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    model      coin       metric\n",
       "0        LinearRegression   bnb-usd    17.002522\n",
       "1   RandomForestRegressor   bnb-usd   140.459650\n",
       "2        LinearRegression   xmr-usd    13.235198\n",
       "3   RandomForestRegressor   xmr-usd    27.408966\n",
       "4        LinearRegression   btc-usd    69.574715\n",
       "5   RandomForestRegressor   btc-usd  3606.136179\n",
       "6        LinearRegression   xrp-usd    12.317904\n",
       "7   RandomForestRegressor   xrp-usd    32.442800\n",
       "8        LinearRegression   dot-usd     4.093729\n",
       "9   RandomForestRegressor   dot-usd     8.639492\n",
       "10       LinearRegression  link-usd    13.634672\n",
       "11  RandomForestRegressor  link-usd    37.788630\n",
       "12       LinearRegression  aave-usd     3.579303\n",
       "13  RandomForestRegressor  aave-usd     9.341335\n",
       "14       LinearRegression  uni1-usd     2.036294\n",
       "15  RandomForestRegressor  uni1-usd     9.080294\n",
       "16       LinearRegression   vet-usd    14.310090\n",
       "17  RandomForestRegressor   vet-usd    38.027085\n",
       "18       LinearRegression   fil-usd     8.626917\n",
       "19  RandomForestRegressor   fil-usd    32.508731\n",
       "20       LinearRegression   zil-usd   132.322583\n",
       "21  RandomForestRegressor   zil-usd    26.767661\n",
       "22       LinearRegression   lrc-usd    41.360237\n",
       "23  RandomForestRegressor   lrc-usd   400.403597\n",
       "24       LinearRegression  doge-usd    14.933210\n",
       "25  RandomForestRegressor  doge-usd   182.308127\n",
       "26       LinearRegression   rep-usd     8.625588\n",
       "27  RandomForestRegressor   rep-usd    12.635247\n",
       "28       LinearRegression   icp-usd     0.707277\n",
       "29  RandomForestRegressor   icp-usd     7.706200\n",
       "30       LinearRegression   ltc-usd    34.609208\n",
       "31  RandomForestRegressor   ltc-usd   225.638346\n",
       "32       LinearRegression   ada-usd    20.065167\n",
       "33  RandomForestRegressor   ada-usd   180.101381\n",
       "34       LinearRegression   axs-usd     2.491264\n",
       "35  RandomForestRegressor   axs-usd     8.714931"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_pure = {\"model\": [], \"coin\": [], \"metric\": []}\n",
    "\n",
    "for coin in tqdm(coins_all):\n",
    "    flag = False\n",
    "    while not flag:\n",
    "        try:\n",
    "            #  reading all the data\n",
    "            coin_data = spark.read.option(\"mode\", \"DROPMALFORMED\").csv(coins_meta[coin], header=True, inferSchema=True).drop(\"Dividends\", \"Stock Splits\")\n",
    "            coin_data = coin_data.withColumn(\"Date\", F.to_timestamp(coin_data.Date, \"yyyy-MM-dd\"))\n",
    "\n",
    "            #  transformation to train / test\n",
    "            train, test, features_scaler, target_scaler = create_train_test(coin_data, \"Date\", \"Close\", 1, 0.2, scale=True)\n",
    "#             target_scaler.write().overwrite().save((hdfs_scalers_dir / coin).as_posix())\n",
    "            train.cache()\n",
    "            test.cache()\n",
    "\n",
    "            #  evaluation of the models\n",
    "            for model, model_params in zip(models, models_params):\n",
    "                model, metric, preds = eval_model(model(**model_params), train, test, get_mae)\n",
    "\n",
    "                model_type = model.uid.split('_')[0]\n",
    "                results_pure[\"model\"].append(model_type)\n",
    "                results_pure[\"coin\"].append(coin)\n",
    "                results_pure[\"metric\"].append(metric)\n",
    "#                 model.write().overwrite().save((hdfs_models_dir / f\"{model_type}_{coin}\").as_posix())\n",
    "#                 preds.select(\"target\", \"prediction\").write.mode('overwrite').parquet((hdfs_preds_dir / f\"{model_type}_{coin}.parquet\").as_posix())\n",
    "\n",
    "            train.unpersist()\n",
    "            test.unpersist()\n",
    "            flag = True\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "    \n",
    "results_pure = pd.DataFrame(results_pure)\n",
    "results_pure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "regular-destiny",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LinearRegression</th>\n",
       "      <td>22.973660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestRegressor</th>\n",
       "      <td>277.006036</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           metric\n",
       "model                            \n",
       "LinearRegression        22.973660\n",
       "RandomForestRegressor  277.006036"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_pure.groupby(\"model\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fiscal-companion",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.createDataFrame(results_pure).write.mode('overwrite').csv((hdfs_data_dir / \"results_pure.csv\").as_posix())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "comparable-infrared",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "banned-shareware",
   "metadata": {},
   "source": [
    "results analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "occupational-owner",
   "metadata": {},
   "outputs": [],
   "source": [
    "res0 = results.copy()\n",
    "res1 = results_pure.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "framed-emission",
   "metadata": {},
   "outputs": [],
   "source": [
    "res0[\"type\"]= \"media\"\n",
    "res1[\"type\"] = \"pure\"\n",
    "res = pd.concat((res, res1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "prompt-motorcycle",
   "metadata": {},
   "outputs": [],
   "source": [
    "res[\"model_type\"] = res[\"model\"] + \"_\" + res[\"type\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "hybrid-extraction",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "sorted-regulation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz0AAAFzCAYAAAAUpWdAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABCM0lEQVR4nO3de3hU1b34//ciIKgggoAHsRU8RQWSEMJd5H61iAoWrSKXWlqVKvYiRW0FsXra/uRrvaDSegGreEhFUb9HqyjiDTkgkUhVBLQNqOWnCEIFRAms7x8zpFwChiRDyPB+PU+ezOzZa+3PXrPn8pm11t4hxogkSZIkpatqlR2AJEmSJKWSSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRWvbIDKI0GDRrEpk2bVnYYkiRJkg5R+fn5n8UYG5b0WJVIepo2bcrixYsrOwxJkiRJh6gQwqp9PebwNkmSJElpzaRHkiRJUloz6ZEkSZKU1qrEnB5JkiT927Zt2/joo4/YunVrZYciHXS1atXixBNPpEaNGqUuY9IjSZJUxXz00UfUqVOHpk2bEkKo7HCkgybGyLp16/joo49o1qxZqcs5vE2SJKmK2bp1K8cdd5wJjw47IQSOO+64A+7lNOmRJEmqgkx4dLgqy7Fv0iNJkiQprZn0SJIkSUlNmzbls88+K/M6GzZs4O67705FaCoHkx5JkiSpgpj0HJpMeiRJklSlFRYWctpppzF69GgyMzMZNmwYL7zwAl26dKF58+YsWrSI9evXc+6555KdnU2nTp1YunQpAOvWraNfv360adOGSy+9lBhjcb0PP/wwHTp0ICcnh0svvZTt27d/YyzXXHMNH3zwATk5OYwbN47hw4fz5JNPFj8+bNgwnnrqKaZPn84555zDgAEDOPXUU5k0aVK5tqv9M+mRJElSlff+++9z1VVXsXTpUt577z0eeeQRXnvtNSZPnsx//dd/MXHiRNq0acPSpUv5r//6L0aMGAHApEmTOOOMM1iyZAlnn302q1evBmDZsmXk5eUxf/58CgoKyMjIYMaMGd8Yx+9+9zv+8z//k4KCAm655RZGjx7NtGnTANi4cSOvv/463/3udwFYtGgRM2bMoKCggEcffZTFixeXebvaP6/TI0mSpCqvWbNmZGVlAdCqVSt69+5NCIGsrCwKCwtZtWoVjz32GAC9evVi3bp1bNy4kVdeeYXHH38cgIEDB1KvXj0A5s6dS35+Pu3btwfgyy+/pFGjRgccV/fu3fnJT37Cp59+yuOPP855551H9eqJr+B9+/bluOOOA2DIkCG89tprVK9evUK2q92Z9EiStIcNz95ZrvLHDriygiKRVFo1a9Ysvl2tWrXi+9WqVaOoqKg40djVzlMfl3QK5BgjI0eO5Le//W25Yxs+fDgzZsxg5syZPPDAA3ttf9f7Fbld/ZvD2yRJkpT2unXrVjxM7KWXXqJBgwYcc8wxuy3/61//yueffw5A7969mTVrFp9++ikA69evZ9WqVd+4nTp16vDFF1/stmzUqFHcdtttQKIXaqfnn3+e9evX8+WXX/LEE0/QpUuXMm9X+2dPjyRJktLeDTfcwA9+8AOys7M56qijePDBBwGYOHEiF154Ibm5uXTv3p1vf/vbALRs2ZKbbrqJfv36sWPHDmrUqMFdd93FSSedtN/tHHfccXTp0oXMzEzOPPNMbrnlFo4//nhatGjBueeeu9u6Z5xxBsOHD+f999/noosuol27dgBl2q72L+x6hopDVbt27eLixYsrOwxJ0mHC4W061C1btowWLVpUdhgqpS1btpCVlcWbb75J3bp1AZg+fTqLFy9mypQplRxd1VTSayCEkB9jbFfS+g5vkyRJklLkhRde4LTTTuPKK68sTnh08Dm8TZIkSTpA69ato3fv3nstnzt3bvEZ2QD69OlTfBrsXY0aNYpRo0alMkTtwqRHkiRJOkDHHXccBQUFlR2GSsnhbZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUprnshAkiSpipu1aG2F1ve9Dg2/cZ3atWuzadOm3ZZNnTqVo446ihEjRlRoPHtq2rQpderUIYRAvXr1+POf/3zIXLzzYLVBWe183v75z38yduxYZs2aVdkhHRQmPZIkSaoQl112WUrrjzESYwRg3rx5NGjQgIkTJ3LTTTdx7733Vkjd1aqVbyBUqtugopxwwgmHTcIDDm+TJElSBbnhhhuYPHkyAD169GD8+PF06NCBU045hVdffRWA7du3M27cONq3b092djZ//OMfAdi0aRO9e/cmNzeXrKwsnnzySQAKCwtp0aIFY8aMITc3lw8//HC3bXbu3JmPP/4YgLVr13LeeefRvn172rdvz/z584uX9+3bl9zcXC699FJOOukkPvvssxLrvuWWW4pjmzhxIgCbN29m4MCBtG7dmszMTPLy8gC45ppraNmyJdnZ2Vx99dV7tUFBQQGdOnUiOzubwYMH8/nnn++3bUoyffp0zj33XAYNGkSzZs2YMmUKt956K23atKFTp06sX78egA8++IABAwbQtm1bunbtynvvvQfAP/7xDzp37kz79u25/vrri+stLCwkMzOz+HbXrl3Jzc0lNzeX119//QCf+UOfSY8kSZJSoqioiEWLFnHbbbcxadIkAO6//37q1q3LG2+8wRtvvMG9997LP/7xD2rVqsXs2bN58803mTdvHr/4xS+Ke3WWL1/OiBEjWLJkyV7D2J599lnOPfdcAK666ip+9rOf8cYbb/DYY48xevRoACZNmkSvXr148803GTx48G4XC9217uXLl7Ny5UoWLVpEQUEB+fn5vPLKKzz77LOccMIJvPXWW7z99tsMGDCA9evXM3v2bN555x2WLl3Kr3/96732f8SIEfz+979n6dKlZGVlFbfBvtpmX95++20eeeQRFi1axK9+9SuOOuoolixZQufOnfnzn/8MwI9//GPuvPNO8vPzmTx5MmPGjCluk8svv5w33niD//iP/yix/kaNGvH888/z5ptvkpeXx9ixY/cbT1Xk8DZJkiSlxJAhQwBo27YthYWFAMyZM4elS5cWD63auHEjK1eu5MQTT+S6667jlVdeoVq1anz88cd88sknAJx00kl06tRpt7p79uzJJ598QqNGjbjpppsAeOGFF3j33XeL1/nXv/7FF198wWuvvcbs2bMBGDBgAPXq1SteZ9e658yZw5w5c2jTpg2Q6H1auXIlXbt25eqrr2b8+PGcddZZdO3alaKiImrVqsXo0aMZOHAgZ5111m7xbdy4kQ0bNtC9e3cARo4cydChQ/fbNvvSs2dP6tSpQ506dahbty6DBg0CICsri6VLl7Jp0yZef/313er/6quvAJg/fz6PPfYYAMOHD2f8+PF71b9t2zauuOIKCgoKyMjIYMWKFfuNpyoy6ZEkSVJK1KxZE4CMjAyKioqAxNyZO++8k/79+++27vTp01m7di35+fnUqFGDpk2bsnXrVgCOPvroveqeN28eRx99NKNGjWLChAnceuut7NixgwULFnDkkUfutu7OHqOS7Fp3jJFrr72WSy+9dK/18vPzeeaZZ7j22mvp168fEyZMYNGiRcydO5eZM2cyZcoUXnzxxVK2TMlt803rAlSrVq34frVq1SgqKmLHjh0ce+yxFBQUlFg+hLDf+v/whz9w/PHH89Zbb7Fjxw5q1apV6v2oKkx6JElShdjw7J3lKn/sgCsrKBIdyvr3788999xDr169qFGjBitWrKBJkyZs3LiRRo0aUaNGDebNm8eqVau+sa4jjzyS2267jaysLH7961/Tr18/pkyZwrhx44DEnJqcnBzOOOMM/vKXvzB+/HjmzJlTPLempNiuv/56hg0bRu3atfn444+pUaMGRUVF1K9fn4svvpjatWszffp0Nm3axJYtW/jud79Lp06d+M53vrNbXXXr1qVevXq8+uqrdO3alYceeqi416eiHXPMMTRr1oxHH32UoUOHEmNk6dKltG7dmi5dujBz5kwuvvhiZsyYUWL5jRs3cuKJJ1KtWjUefPBBtm/fnpI4K5NJjyRJUhVXmlNMV7QtW7Zw4oknFt//+c9/Xqpyo0ePprCwkNzcXGKMNGzYkCeeeIJhw4YxaNAg2rVrR05ODqeddlqp6mvcuDEXXnghd911F3fccQc/+clPyM7OpqioiG7dujF16lQmTpzIhRdeSF5eHt27d6dx48bUqVNnr1Nu9+vXj2XLltG5c2cgcXrnhx9+mPfff59x48ZRrVo1atSowT333MMXX3zBOeecw9atW4kx8oc//GGv2B588EEuu+wytmzZwsknn8y0adNKtU9lMWPGDC6//HJuuukmtm3bxve//31at27N7bffzkUXXcTtt9/OeeedV2LZMWPGcN555/Hoo4/Ss2fPEnvWqrqwv+6+Q0W7du3i4sWLKzsMSdJhwh6LsrHdDp5ly5bRokWLyg6jyvjqq6/IyMigevXqLFiwgMsvv3yfQ8FUNZT0Gggh5McY25W0vj09kiRJSmurV6/m/PPPZ8eOHRxxxBHlvqaPqh6THkmSJKW15s2bs2TJksoOY7+ee+65vc6s1qxZs+Kzzql8THokSZKkSta/f/+9zminiuPFSSVJkiSlNZMeSZIkSWnNpEeSJElSWnNOjyRJUhVX3tOF78nThyvd2NMjSZKkA5aRkUFOTg6ZmZkMGjSIDRs2VEi906dP54orrqiQupo2bUpWVhY5OTnk5OTw+uuvV0i9eyooKOCZZ54pvj99+nQaNmxYfJHVki5cKhg1ahSzZs0CEhetfffdd1O2LZMeSZIkHbAjjzySgoIC3n77berXr89dd91V2SGVaN68eRQUFFBQUMDpp59eqjJFRUUHtI09kx6ACy64gIKCAubPn8/NN9/Mhx9+eEB1VkRcZRVjZMeOHQdlWzvdd999tGzZMmX1m/RIkiSpXDp37szHH38MwKJFizj99NNp06YNp59+OsuXLwcSvR9DhgxhwIABNG/enF/+8pfF5adNm8Ypp5xC9+7dmT9/fvHyVatW0bt3b7Kzs+nduzerV68GEj0El19+OT179uTkk0/m5Zdf5pJLLqFFixaMGjVqv7Hur86f//zn9OzZk/Hjx/PBBx8wYMAA2rZtS9euXXnvvfcAePTRR8nMzKR169Z069aNr7/+mgkTJpCXl0dOTg55eXm7be+4447jO9/5DmvWrAHg4YcfpkOHDuTk5HDppZeyfft2AO6//35OOeUUevTowY9+9KPi3q6yxgXwzjvvFG8rOzublStXAnDrrbeSmZlJZmYmt912GwCFhYW0aNGCMWPGkJubu88krXbt2owfP562bdvSp08fFi1aRI8ePTj55JN56qmnANi+fTvjxo2jffv2ZGdn88c//hFIJFNXXHEFLVu2ZODAgXz66afF9fbo0YPFixcDcPnll9OuXTtatWrFxIkT9/t8lpZzeiRJklRm27dvZ+7cufzwhz8E4LTTTuOVV16hevXqvPDCC1x33XU89thjQKJHZMmSJdSsWZNTTz2VK6+8kurVqzNx4kTy8/OpW7cuPXv2pE2bNgBcccUVjBgxgpEjR/LAAw8wduxYnnjiCQA+//xzXnzxRZ566ikGDRrE/Pnzue+++2jfvj0FBQXk5OQA0LNnTzIyMqhZsyYLFy7cb50rVqzghRdeICMjg969ezN16lSaN2/OwoULGTNmDC+++CI33ngjzz33HE2aNGHDhg0cccQR3HjjjSxevJgpU6YAiQRvp9WrV7N161ays7NZtmwZeXl5zJ8/nxo1ajBmzBhmzJhBnz59+M1vfsObb75JnTp16NWrF61bty6uoyxxAUydOpWrrrqKYcOG8fXXX7N9+3by8/OZNm0aCxcuJMZIx44d6d69O/Xq1WP58uVMmzaNu+++e5/P9+bNm+nRowe///3vGTx4ML/+9a95/vnneffddxk5ciRnn302999/P3Xr1uWNN97gq6++okuXLvTr148lS5awfPly/va3v/HJJ5/QsmVLLrnkkr22cfPNN1O/fn22b99O7969Wbp0KdnZ2Qd8bO7KpEeSJEkH7MsvvyQnJ4fCwkLatm1L3759Adi4cSMjR45k5cqVhBDYtm1bcZnevXtTt25dAFq2bMmqVav47LPP6NGjBw0bNgQSw8JWrFgBwIIFC3j88ccBGD58+G69Q4MGDSKEQFZWFscffzxZWVkAtGrVisLCwuKkZ968eTRo0KC43P7qHDp0KBkZGWzatInXX3+doUOHFj/21VdfAdClSxdGjRrF+eefz5AhQ/bZPnl5ecybN4/ly5dz7733UqtWLebOnUt+fj7t27cvbsNGjRqxaNEiunfvTv369Yvj2NkG5Ymrc+fO3HzzzXz00UcMGTKE5s2b89prrzF48GCOPvpoAIYMGcKrr77K2WefzUknnUSnTp32uU8ARxxxBAMGDAAgKyuLmjVrUqNGDbKysigsLARgzpw5LF26tHi+zsaNG1m5ciWvvPIKF154IRkZGZxwwgn06tWrxG385S9/4U9/+hNFRUWsWbOGd99999BPekIIGcBi4OMY41khhPpAHtAUKATOjzF+nuo4JEmSVHF2zunZuHEjZ511FnfddRdjx47l+uuvp2fPnsyePZvCwkJ69OhRXKZmzZrFtzMyMornqIQQSrXNXdfbWVe1atV2q7datWoHNPdl1zp3JgI7duzg2GOPpaCgYK/1p06dysKFC3n66afJyckpcR1IJG9TpkxhwYIFDBw4kDPPPJMYIyNHjuS3v/3tbuvOnj17vzGWNa6LLrqIjh078vTTT9O/f3/uu+8+YozfuJ39qVGjRnGb7dr2u7Z7jJE777yT/v3771b2mWee+cbn+h//+AeTJ0/mjTfeoF69eowaNYqtW7d+Y1zf5GDM6bkKWLbL/WuAuTHG5sDc5H1JkiSV0bEDrqzQvwNRt25d7rjjDiZPnsy2bdvYuHEjTZo0AXYf5rUvHTt25KWXXmLdunVs27aNRx99tPix008/nZkzZwIwY8YMzjjjjAOKrSSlqfOYY46hWbNmxbHEGHnrrbcA+OCDD+jYsSM33ngjDRo04MMPP6ROnTp88cUXJW6vc+fODB8+nNtvv53evXsza9as4rks69evZ9WqVXTo0IGXX36Zzz//nKKiouLhgOWN6+9//zsnn3wyY8eO5eyzz2bp0qV069aNJ554gi1btrB582Zmz55N165dy9Gie+vfvz/33HNPcS/fihUr2Lx5M926dWPmzJls376dNWvWMG/evL3K/utf/+Loo4+mbt26fPLJJ/z1r3+tkJhSmvSEEE4EBgL37bL4HODB5O0HgXNTGYMkSZJSq02bNrRu3ZqZM2fyy1/+kmuvvZYuXboUT9Lfn8aNG3PDDTfQuXNn+vTpQ25ubvFjd9xxB9OmTSM7O5uHHnqI22+/vdyxlrbOGTNmcP/999O6dWtatWrFk08+CcC4cePIysoiMzOTbt260bp1a3r27Mm7775b4okMAMaPH8+0adP41re+xU033US/fv3Izs6mb9++rFmzhiZNmnDdddfRsWNH+vTpQ8uWLYuHAZYnrry8PDIzM8nJyeG9995jxIgR5ObmMmrUKDp06EDHjh0ZPXp08RyqijJ69GhatmxJbm4umZmZXHrppRQVFTF48GCaN29OVlYWl19+Od27d9+rbOvWrWnTpg2tWrXikksuoUuXLhUSU9hfF1e5Kw9hFvBboA5wdXJ424YY47G7rPN5jLFeCWV/DPwY4Nvf/nbbVatWpSxOSZJ2Vd4LPR6uF3a03Q6eZcuW0aJFi8oOQxVo06ZN1K5duzg5uOSSSxg8eHBlh3XIKuk1EELIjzG2K2n9lPX0hBDOAj6NMeaXpXyM8U8xxnYxxnY7J7ZJkiRJ6eiGG24ovthrs2bNOPfccys7pLSSyhMZdAHODiF8F6gFHBNCeBj4JITQOMa4JoTQGPh0v7VIkiRJaW7y5MmVHcJuOnbsWHxmuJ0eeuih4rPkVTUpS3pijNcC1wKEEHqQGN52cQjhFmAk8Lvk/ydTFYMkSZKkA7dw4cLKDqFCHYyzt+3pd0DfEMJKoG/yviRJkiSlxEG5OGmM8SXgpeTtdUDvg7FdSZIkSaqMnh5JkiRJOmgOSk+PJEmSUuePK2dWaH2XNv9+hdYnVTZ7eiRJknTAateuvdeyqVOn8uc//znl227atClZWVlkZ2fTvXt3DqXrOR6sNtCBsadHkiRJFeKyyy5Laf0xRmKMAMybN48GDRowceJEbrrpJu69994KqbtatfL1CaS6DQ5UUVER1av7ld+eHkmSJFWIG264ofh6Mz169GD8+PF06NCBU045hVdffRWA7du3M27cONq3b092djZ//OMfAdi0aRO9e/cmNzeXrKwsnnwycVWTwsJCWrRowZgxY8jNzeXDDz/cbZudO3fm448/BmDt2rWcd955tG/fnvbt2zN//vzi5X379iU3N5dLL72Uk046ic8++6zEum+55Zbi2CZOnAjA5s2bGThwIK1btyYzM5O8vDwArrnmGlq2bEl2djZXX331Xm1QUFBAp06dyM7OZvDgwXz++ef7bZuSTJ8+nXPOOYcBAwZw6qmnMmnSpOJ2yczMLF5v8uTJ3HDDDcX1X3fddXTv3p3bb7+d/Px8unfvTtu2benfvz9r1qw54Oe2qjPtkyRJUkoUFRWxaNEinnnmGSZNmsQLL7zA/fffT926dXnjjTf46quv6NKlC/369eNb3/oWs2fP5phjjuGzzz6jU6dOnH322QAsX76cadOmcffdd++1jWeffZZzzz0XgKuuuoqf/exnnHHGGaxevZr+/fuzbNkyJk2aRK9evbj22mt59tln+dOf/lRcfte658yZw8qVK1m0aBExRs4++2xeeeUV1q5dywknnMDTTz8NwMaNG1m/fj2zZ8/mvffeI4TAhg0b9optxIgR3HnnnXTv3p0JEyYwadIkbrvttn22zb4sWrSIt99+m6OOOor27dszcOBAGjRosN+237BhAy+//DLbtm2je/fuPPnkkzRs2JC8vDx+9atf8cADD+y3fLox6ZEkSVJKDBkyBIC2bdtSWFgIwJw5c1i6dCmzZs0CEgnEypUrOfHEE7nuuut45ZVXqFatGh9//DGffPIJACeddBKdOnXare6ePXvyySef0KhRI2666SYAXnjhBd59993idf71r3/xxRdf8NprrzF79mwABgwYQL169YrX2bXuOXPmMGfOHNq0aQMkep9WrlxJ165dufrqqxk/fjxnnXUWXbt2paioiFq1ajF69GgGDhzIWWedtVt8GzduZMOGDXTv3h2AkSNHMnTo0P22zb707duX4447rrjca6+9Vpzo7csFF1wAJJK6t99+m759+wKJnrbGjRvvt2w6MumRJElSStSsWROAjIwMioqKgMTcmTvvvJP+/fvvtu706dNZu3Yt+fn51KhRg6ZNm7J161YAjj766L3qnjdvHkcffTSjRo1iwoQJ3HrrrezYsYMFCxZw5JFH7rbuznlAJdm17hgj1157LZdeeule6+Xn5/PMM89w7bXX0q9fPyZMmMCiRYuYO3cuM2fOZMqUKbz44oulbJmS22ZfQgh73a9evTo7duwoXrazrfbcrxgjrVq1YsGCBaWOLR2Z9EiSdAjZNCuvXOVrf++CCopEVUlVOsV0//79ueeee+jVqxc1atRgxYoVNGnShI0bN9KoUSNq1KjBvHnzSnVGtiOPPJLbbruNrKwsfv3rX9OvXz+mTJnCuHHjgMScmpycHM444wz+8pe/MH78eObMmVM8t6ak2K6//nqGDRtG7dq1+fjjj6lRowZFRUXUr1+fiy++mNq1azN9+nQ2bdrEli1b+O53v0unTp34zne+s1tddevWpV69erz66qt07dqVhx56qLjX50A9//zzrF+/niOPPJInnniCBx54gOOPP55PP/2UdevWUbt2bf7nf/6HAQMG7FX21FNPZe3atSxYsIDOnTuzbds2VqxYQatWrcoUS1Vl0iNJkqQDtmXLFk488cTi+z//+c9LVW706NEUFhaSm5tLjJGGDRvyxBNPMGzYMAYNGkS7du3IycnhtNNOK1V9jRs35sILL+Suu+7ijjvu4Cc/+QnZ2dkUFRXRrVs3pk6dysSJE7nwwgvJy8uje/fuNG7cmDp16rBp06bd6urXrx/Lli2jc+fOQOK03A8//DDvv/8+48aNo1q1atSoUYN77rmHL774gnPOOYetW7cSY+QPf/jDXrE9+OCDXHbZZWzZsoWTTz6ZadOmlWqf9nTGGWcwfPhw3n//fS666CLatWsHwIQJE+jYsSPNmjXbZ3sdccQRzJo1i7Fjx7Jx40aKior46U9/etglPWF/3X2Hinbt2sXFixdXdhiSpMPEhmfvLFf5YwdcWeayVbmnpzLb7XCzbNkyWrRoUdlhVBlfffUVGRkZVK9enQULFnD55ZdTUFBQ2WGVyvTp01m8eDFTpkyp7FAOKSW9BkII+THGdiWtb0+PJEmS0trq1as5//zz2bFjB0cccUS5r+mjqsekR2VWnl/0/DVPkiQdLM2bN2fJkiWVHcZ+Pffcc4wfP363Zc2aNWP27NmMGjWqcoJKIyY9kiRJUiXr37//Xme0U8WpVtkBSJIkSVIqmfRIkiRJSmsmPZIkSZLSmnN6JEmSqrjynup8T17kVunGnh5JkiQdsIyMDHJycsjMzGTQoEFs2LChQuqdPn06V1xxRYXU1bRpU7KyssjJySEnJ4fXX3+9QurdU0FBAc8880zx/enTp9OwYcPii6yWdOFSHVwmPZIkSTpgRx55JAUFBbz99tvUr1+fu+66q7JDKtG8efMoKCigoKCA008/vVRlioqKDmgbeyY9ABdccAEFBQXMnz+fm2++mQ8//PCA6qyIuMoqxsiOHTsqvN6DFX9JTHokSZJULp07d+bjjz8GYNGiRZx++um0adOG008/neXLlwOJ3o8hQ4YwYMAAmjdvzi9/+cvi8tOmTeOUU06he/fuzJ8/v3j5qlWr6N27N9nZ2fTu3ZvVq1cDMGrUKC6//HJ69uzJySefzMsvv8wll1xCixYtvvGaNvur8+c//zk9e/Zk/PjxfPDBBwwYMIC2bdvStWtX3nvvPQAeffRRMjMzad26Nd26dePrr79mwoQJ5OXlkZOTQ17e7kMNjzvuOL7zne+wZs0aAB5++GE6dOhATk4Ol156Kdu3bwfg/vvv55RTTqFHjx786Ec/Ku7tKmtcAO+8807xtrKzs1m5ciUAt956K5mZmWRmZnLbbbcBUFhYSIsWLRgzZgy5ubn7TNJq167NL37xC3Jzc+nduzdr164FoEePHixevBiAzz77jKZNmxY/70OHDmXQoEH069ePzZs3c8kll9C+fXvatGnDk08+ud/nq6I4p0eSJElltn37dubOncsPf/hDAE477TReeeUVqlevzgsvvMB1113HY489BiR6RJYsWULNmjU59dRTufLKK6levToTJ04kPz+funXr0rNnT9q0aQPAFVdcwYgRIxg5ciQPPPAAY8eO5YknngDg888/58UXX+Spp55i0KBBzJ8/n/vuu4/27dtTUFBATk4OAD179iQjI4OaNWuycOHC/da5YsUKXnjhBTIyMujduzdTp06lefPmLFy4kDFjxvDiiy9y44038txzz9GkSRM2bNjAEUccwY033sjixYuZMmUKkPiiv9Pq1avZunUr2dnZLFu2jLy8PObPn0+NGjUYM2YMM2bMoE+fPvzmN7/hzTffpE6dOvTq1YvWrVsX11GWuACmTp3KVVddxbBhw/j666/Zvn07+fn5TJs2jYULFxJjpGPHjnTv3p169eqxfPlypk2bxt13373P53vz5s3k5ubyf/7P/+HGG29k0qRJxfu9LwsWLGDp0qXUr1+f6667jl69evHAAw+wYcMGOnToQJ8+fTj66KNLdbyVlUmPJEmSDtiXX35JTk4OhYWFtG3blr59+wKwceNGRo4cycqVKwkhsG3btuIyvXv3pm7dugC0bNmSVatW8dlnn9GjRw8aNmwIJIaFrVixAkh8WX788ccBGD58+G69Q4MGDSKEQFZWFscffzxZWVkAtGrVisLCwuKkZ968eTRo0KC43P7qHDp0KBkZGWzatInXX3+doUOHFj/21VdfAdClSxdGjRrF+eefz5AhQ/bZPnl5ecybN4/ly5dz7733UqtWLebOnUt+fj7t27cvbsNGjRqxaNEiunfvTv369Yvj2NkG5Ymrc+fO3HzzzXz00UcMGTKE5s2b89prrzF48ODiJGPIkCG8+uqrnH322Zx00kl06tRpn/sEUK1aNS64IHGii4svvni/bbBT3759i/dtzpw5PPXUU0yePBmArVu3snr1alq0aPGN9ZSHSY8kSZIO2M45PRs3buSss87irrvuYuzYsVx//fX07NmT2bNnU1hYSI8ePYrL1KxZs/h2RkZG8RyPEEKptrnrejvrqlat2m71VqtW7YDmjuxa585EYMeOHRx77LEUFBTstf7UqVNZuHAhTz/9NDk5OSWuA4nkbcqUKSxYsICBAwdy5plnEmNk5MiR/Pa3v91t3dmzZ+83xrLGddFFF9GxY0eefvpp+vfvz3333UeM8Ru3cyB2tl/16tWL5wFt3bp1n/XGGHnsscc49dRTD3hb5eGcHkmSpCqu9vcuqNC/A1G3bl3uuOMOJk+ezLZt29i4cSNNmjQBdh/mtS8dO3bkpZdeYt26dWzbto1HH320+LHTTz+dmTNnAjBjxgzOOOOMA4qtJKWp85hjjqFZs2bFscQYeeuttwD44IMP6NixIzfeeCMNGjTgww8/pE6dOnzxxRclbq9z584MHz6c22+/nd69ezNr1iw+/fRTANavX8+qVavo0KEDL7/8Mp9//jlFRUXFwwHLG9ff//53Tj75ZMaOHcvZZ5/N0qVL6datG0888QRbtmxh8+bNzJ49m65du5a6/Xbs2MGsWbMAeOSRR4rbr2nTpuTn5wMUP16S/v37c+eddxYnX0uWLCn1tsvDpEeSJEnl0qZNG1q3bs3MmTP55S9/ybXXXkuXLl2KJ+nvT+PGjbnhhhvo3Lkzffr0ITc3t/ixO+64g2nTppGdnc1DDz3E7bffXu5YS1vnjBkzuP/++2ndujWtWrUqnnA/btw4srKyyMzMpFu3brRu3ZqePXvy7rvvlngiA4Dx48czbdo0vvWtb3HTTTfRr18/srOz6du3L2vWrKFJkyZcd911dOzYkT59+tCyZcviYYDliSsvL4/MzExycnJ47733GDFiBLm5uYwaNYoOHTrQsWNHRo8eXTyHqjSOPvpo3nnnHdq2bcuLL77IhAkTALj66qu55557OP300/nss8/2Wf76669n27ZtZGdnk5mZyfXXX1/qbZdH2F8X16GiXbt2cefZIHTo2PDsnWUue+yAKyswEkmqWOV5f4PyvceV9yKTlXlRycpst8PNsmXLUj4HQgfXpk2bqF27NkVFRQwePJhLLrmEwYMHV3ZYe6lduzabNm2q7DBKfA2EEPJjjO1KWt+eHkmSJKmS3XDDDcUXe23WrBnnnntuZYeUVjyRgSRJklTJdp7N7FDRsWPH4jPD7fTQQw8dEr08ZWHSI0mSVAXFGEt91jPpQC1cuLCyQ9inskzPcXibJElSFVOrVi3WrVtXpi9/UlUWY2TdunXUqlXrgMrZ0yNJklTFnHjiiXz00UesXbu2skORDrpatWpx4oknHlAZkx5JkqQqpkaNGjRr1qyyw5CqDIe3SZIkSUprJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUpr1Ss7AKksNs3KK3PZ2t+7oAIjkSQdCsrzuQB+Nkjpzp4eSZIkSWnNpEeSJElSWjPpkSRJkpTWTHokSZIkpTWTHkmSJElpzaRHkiRJUloz6ZEkSZKU1kx6JEmSJKU1kx5JkiRJac2kR5IkSVJaM+mRJEmSlNZMeiRJkiSlNZMeSZIkSWktZUlPCKFWCGFRCOGtEMI7IYRJyeX1QwjPhxBWJv/XS1UMkiRJkpTKnp6vgF4xxtZADjAghNAJuAaYG2NsDsxN3pckSZKklKieqopjjBHYlLxbI/kXgXOAHsnlDwIvAeNTFcc32fDsneUqf+yAKysoEkmSJEmpkNI5PSGEjBBCAfAp8HyMcSFwfIxxDUDyf6N9lP1xCGFxCGHx2rVrUxmmJEmSpDSW0qQnxrg9xpgDnAh0CCFkHkDZP8UY28UY2zVs2DBlMUqSJElKbwfl7G0xxg0khrENAD4JITQGSP7/9GDEIEmSJOnwlMqztzUMIRybvH0k0Ad4D3gKGJlcbSTwZKpikCRJkqSUncgAaAw8GELIIJFc/SXG+D8hhAXAX0IIPwRWA0NTGIMkSZKkw1wqz962FGhTwvJ1QO9UbVeSJEmSdnVQ5vRIkiRJUmUx6ZEkSZKU1kx6JEmSJKU1kx5JkiRJac2kR5IkSVJaM+mRJEmSlNZMeiRJkiSlNZMeSZIkSWnNpEeSJElSWjPpkSRJkpTWTHokSZIkpTWTHkmSJElprXplB3C42zQrr8xla3/vggqMRJIkSUpP9vRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUprJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktFaqpCeEMDiEUHeX+8eGEM5NWVSSJEmSVEFK29MzMca4ceedGOMGYGJKIpIkSZKkClTapKek9apXZCCSJEmSlAqlTXoWhxBuDSH8Zwjh5BDCH4D8VAYmSZIkSRWhtEnPlcDXQB7wKLAV+EmqgpIkSZKkilKqIWoxxs3ANSmORZIkSZIq3H6TnhDCbTHGn4YQ/i8Q93w8xnh2yiKTJEmSpArwTT09DyX/T051IJIkSZKUCvtNemKM+SGEDOBHMcaLD1JMkiRJklRhvvFEBjHG7UDDEMIRByEeSZIkSapQpb3WTiEwP4TwFLB558IY462pCEqSJEmSKkppk55/Jv+qAXWSy/Y6sYEkSZIkHWpKm/S8G2N8dNcFIYShKYhHkiRJkipUaS9Oem0pl0mSJEnSIeWbrtNzJvBdoEkI4Y5dHjoGKEplYJIkSZJUEb5peNs/gcXA2UD+Lsu/AH6WqqAkSZIkqaJ803V63gLeCiE8klz32zHG5QclMkmSJEmqAKWd0zMAKACeBQgh5CRPXy1JkiRJh7TSJj03AB2ADQAxxgKgaSoCkiRJkqSKVNqkpyjGuDGlkUiSJElSCpT2Oj1vhxAuAjJCCM2BscDrqQtLkiRJkipGaXt6rgRaAV8BjwAbgatSFZQkSZIkVZTSJj0tk3/VgVrAOcAbqQpKkiRJkipKaYe3zQCuBt4GdqQuHEmSJEmqWKVNetbGGP9vSiORJEmSpBQobdIzMYRwHzCXxLweAGKMj6ckKkmSJEmqIKVNen4AnAbU4N/D2yJg0iNJkiTpkFbapKd1jDErpZFIkiRJUgqU9uxt/xtCaJnSSCRJkiQpBUrb03MGMDKE8A8Sc3oCEGOM2SmLTJIkSZIqQGmTngEpjUKSJEmSUqRUSU+McVWqA5EkSZKkVCjtnB5JkiRJqpJMeiRJkiSlNZMeSZIkSWnNpEeSJElSWktZ0hNC+FYIYV4IYVkI4Z0QwlXJ5fVDCM+HEFYm/9dLVQySJEmSlMqeniLgFzHGFkAn4CfJC5xeA8yNMTYH5ibvS5IkSVJKpCzpiTGuiTG+mbz9BbAMaAKcAzyYXO1B4NxUxSBJkiRJB2VOTwihKdAGWAgcH2NcA4nECGi0jzI/DiEsDiEsXrt27cEIU5IkSVIaSnnSE0KoDTwG/DTG+K/Slosx/inG2C7G2K5hw4apC1CSJElSWktp0hNCqEEi4ZkRY3w8ufiTEELj5OONgU9TGYMkSZKkw1sqz94WgPuBZTHGW3d56ClgZPL2SODJVMUgSZIkSdVTWHcXYDjwtxBCQXLZdcDvgL+EEH4IrAaGpjAGSZIkSYe5lCU9McbXgLCPh3unaruSJEmStKuDcvY2SZIkSaosJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUprJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRWvbIDkCSpos1atLZc5ftUUBySpEODPT2SJEmS0ppJjyRJkqS0lhbD28ozjMEhDJIkSVJ6s6dHkiRJUloz6ZEkSZKU1kx6JEmSJKU1kx5JkiRJaS0tTmQgSZJUVW149s5ylT92wJUVFImUvuzpkSRJkpTW7OmRJKmC/XHlzDKXHUaowEgkSWBPjyRJkqQ0Z9IjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktObZ2w5jsxatLVf5PhUUhyRJkpRK9vRIkiRJSmv29JRTea7FAF6PQZIkSUo1e3okSZIkpTWTHkmSJElpzaRHkiRJUlpzTo8kSdJhatOsvHKVr/29CyooEim17OmRJEmSlNbs6VGl8Kx3kqR0Up5r33ndOyn1THokVQkbnr2zzGWPHXBlBUYiSZKqGoe3SZIkSUpr9vRIkiSgfEO0wGFakg5d9vRIkiRJSmsmPZIkSZLSmkmPJEmSpLTmnB7pIPMsZJIkSQeXPT2SJEmS0po9PZIkSdJhojwjTqDqjjqxp0eSJElSWjPpkSRJkpTWUpb0hBAeCCF8GkJ4e5dl9UMIz4cQVib/10vV9iVJkiQJUtvTMx0YsMeya4C5McbmwNzkfUmSJElKmZQlPTHGV4D1eyw+B3gweftB4NxUbV+SJEmS4ODP6Tk+xrgGIPm/0UHeviRJkqTDzCF7IoMQwo9DCItDCIvXrl1b2eFIkiRJqqIOdtLzSQihMUDy/6f7WjHG+KcYY7sYY7uGDRsetAAlSZIkpZeDnfQ8BYxM3h4JPHmQty9JkiTpMJPKU1b/N7AAODWE8FEI4YfA74C+IYSVQN/kfUmSJElKmeqpqjjGeOE+Huqdqm1KUipsmpVXrvK1v3dBBUUiSZLKImVJjySp8m149s4ylz12wJUVGIkkSZXHpEeSlBL2kEmSDhWH7CmrJUmSJKkimPRIkiRJSmsOb5MOI+UZbuRQI0mp9seVM8tcdhihAiOpWmw3VSWV9V3Enh5JkiRJac2kR5IkSVJaM+mRJEmSlNZMeiRJkiSlNZMeSZIkSWnNs7dJkiSpytnw7J3lKn/sgCsrKBJVBSY9kiRJ0gHyMhBVi8PbJEmSJKU1kx5JkiRJac2kR5IkSVJaM+mRJEmSlNY8kYF0gGYtWluu8n0qKA5JkiSVjj09kiRJktKaSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUprJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKaSY8kSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprVWv7AAkld4fV84sV/lhhAqKRJIkqeqwp0eSJElSWjPpkSRJkpTWHN4mKe05LFCSpMObSY+kg2LWorXlKt+nguKQJEmHH4e3SZIkSUprJj2SJEmS0ppJjyRJkqS0ZtIjSZIkKa2Z9EiSJElKayY9kiRJktKap6yWJEmSqpDyXAaivJeAqKrXvrOnR5IkSVJas6dHkg5hXtRVkqTyM+mRJJWoqg5hkCRpTw5vkyRJkpTWTHokSZIkpTWTHkmSJElpzaRHkiRJUloz6ZEkSZKU1kx6JEmSJKU1kx5JkiRJac2kR5IkSVJa8+KkkiRJOux4AebDS6UkPSGEAcDtQAZwX4zxd5URhyRJkirPrEVry1y2TwXGofR30Ie3hRAygLuAM4GWwIUhhJYHOw5JkiRJh4fKmNPTAXg/xvj3GOPXwEzgnEqIQ5IkSdJhoDKSnibAh7vc/yi5TJIkSZIqXIgxHtwNhjAU6B9jHJ28PxzoEGO8co/1fgz8OHn3VGD5QQ209BoAn1V2EFWQ7VY2tlvZ2G5lY7uVnW1XNrZb2dhuZWO7lc2h3G4nxRgblvRAZZzI4CPgW7vcPxH4554rxRj/BPzpYAVVViGExTHGdpUdR1Vju5WN7VY2tlvZ2G5lZ9uVje1WNrZb2dhuZVNV260yhre9ATQPITQLIRwBfB94qhLikCRJknQYOOg9PTHGohDCFcBzJE5Z/UCM8Z2DHYckSZKkw0OlXKcnxvgM8ExlbDsFDvkheIco261sbLeysd3KxnYrO9uubGy3srHdysZ2K5sq2W4H/UQGkiRJknQwVcacHkmSJEk6aEx6DgEhhB4hhP+p7DiqkhDCpsqOoTxCCDeEEK7ez+OjQggnpGjbVbrtDhXp0I4hhHYhhDuSt08LISwIIXy1v2OznNtLm/e6EMKxIYQxKap7VAhhSirqPhhCCGNDCMtCCJ+HEK5JLtvve145tlUYQmhQ0fUejqrae1oI4fVDIIbpIYTvVXYcVVFlHG+VMqcnHYQQqscYiyo7jkNJCCEjxri9suNIE6OAtynhdO5SRYkxLgYWJ++uB8YC51ZaQFXLscAY4O5KjuNQNAY4M8b4j8oOpCoJIQQS0w52VHYsVUGM8fTKjkFVy2HX0xNCaB9CWBpCqBVCODqE8E4I4YoQwsshhL+EEFaEEH4XQhgWQlgUQvhbCOE/k2WnhxBuDSHMA36/R727/YoVQng7hNA0uY2nQwhvJZddkHx8QAjhvRDCa8CQg9kGZbWftpsXQngE+Ftyn98LITyYXHdWCOGoEura7RffEMKUEMKo5O3fhRDeTZafnFzWLPkr9BshhN8crH2uSCGEX4UQlocQXiBxwV1CCDkhhP9N7uvsEEK95K9G7YAZIYSCEMKRe9ST1m0XQngihJCfPL5+nFx2TwhhcXLZpOSyM0MIf9mlXI8Qwv9N3u6X3Oc3QwiPhhBql7CdtGvH5Ovv7V3uX518b3ophPD75HvaihBC1+TjxW0QY/w0xvgGsO0btpG273XJNhqzy/0bQgi/CCGMSz7vS3cef8DvgP9MvkZvKaGu3X4BDslfNUMIjUMIryTLvb3Lc/GD5HPzMtAlpTuaQiGEqcDJwFMhhJ+Fb+ix2tcxm7w9dpfX4czksuNCCHNCCEtCCH8EQur2JvWS+78shHA38CZw/Z7HWvBztURhl56CEMIvQ+L72lshhN8ll70UQrgthPB68rXWYR/1FPcWhkTv90vJ292Tr9OC5PFWJyRMSbbl00Cj1O9pxQl7fL6GEE4KIawMITQIIVQLIbwaEp+fJb6f71FXlTveDrukJ/mh/hRwE/D/AQ+T+EW9NXAVkAUMB06JMXYA7gOu3KWKU4A+McZflHKTA4B/xhhbxxgzgWdDCLWAe4FBQFfgP8q9YwfBftquA/CrGGPL5KqnAn+KMWYD/yLxq1+phBDqA4OBVsnyNyUfuh24J8bYHvj/K2B3DqoQQlsS16RqQ+KLX/vkQ38Gxif39W/AxBjjLBK/vg+LMebEGL8s5TbSpe0uiTG2JZH4jQ0hHEfi+GoHZAPdQwjZwPNApxDC0clyFwB5yQ+vX5N4neaSaMufl3bjadSOe6qefE/7KTAxBfWnw3vdTBLH0U7nA2uB5iTe53KAtiGEbsA1wAfJ1+i4A9jGRcBzMcYcEp87BSGExsAkEslOX6Dlvosf2mKMl5Hooe4JfF7O6q4B2iRfh5cll00EXosxtiHxefTtcm7jUHAqyc8CoAl7H2s71/FztQQhhDNJ9FB3jDG2JvH9ZKejkz1CY4AHDrDqq4GfJF+rXYEvSbTjqSS+K/4IqGq9Tbt9vgKbSPyIPxX4BfBujHEOJbyfl3YDh/LxdtglPUk3kvhgace/XxxvxBjXxBi/Aj4A5iSX/w1oukvZRw9wCNffgD4h8Qti1xjjRuA04B8xxpUxcfq8h8uxLwdbSW23aI9hDB/GGOcnbz8MnHEA9f8L2ArcF0IYAmxJLu8C/Hfy9kNlCbySdQVmxxi3xBj/ReLD+mjg2Bjjy8l1HgS67auCUkiXthsbQngL+F/gWyS+cJ4fQngTWAK0Alomh5c+CwwKIVQHBgJPAp1IfGmcH0IoAEYCJx3A9tOlHff0ePJ/Pru/p1WUKv9eF2NcAjQKIZwQQmhN4kt7NtCPxLH3Jol9al6OzbwB/CDZm5EVY/wC6Ai8FGNcG2P8GsgrR/3pZCmJHu+LgZ3DybuRPI5ijE9T/sTqULAqxvi/JI6zfR1rfq7uWx9gWoxxC0CMcf0uj/13ctkrwDEhhGMPoN75wK0hhLEkPquLSBx//x1j3B5j/CfwYkXswEG01+drjPE+oA6JHxZ29uKX9H5eWofs8Xa4Jj31gdoknuRayWVf7fL4jl3u72D3uU+bAUIIP9ml2/MEEm/Iu7ZnLYAY4wqgLYkD6LchhAnJx6vqucJLarvNe6yz577FEELHXdrrbPbdXkUkfuV6jMQvN7v+ulBV22ynMsV/OLVdCKEHiQ+wzslf7JYALUi8EfdO/mr0NP8+9vJI/Brfi8QPF1+QGO7yfPIX+JwYY8sY4w8Pk3YscZ+Sdr6nbecA5nMehu91s4DvkejxmUniePrtLsfTd2KM9+9ZKIRw8852Si4qbqcQQgCOgOIvX92Aj4GHQggjkutXtXYqkxDCt3Y5ni5j/8fsQOAuEsdVfvLHDUi/ttr5Gbq/Y83P1X0L7Hs/Smq355Jtdl9y2a7tVnz8xRh/B4wGjgT+N4Rw2j7qrBL28flaKySGSp6YXK02lPx+ng7H2+Ga9PwJuB6YwR5zc0orxnjXLm9M/wQKgVyAEEIu0Cx5+wRgS4zxYWBycp33gGYhOVcIuLAc+3Kwlabtvh1C6Jy8fSGJoQgLd2mvp4BVQMsQQs0QQl2gN0BIzL2om7yA7U9JdPFD4heX7ydvD6vgfToYXgEGhxCODCHUITHcZzPweUiO6ScxrHJnr88XJBJLDrO2qwt8HmPckvyA6QQcQ6KtNoYQjgfO3GX9l0i8pn7Ev38d/1+gSwjhOwAhhKNCCKccJu34CYmeiuNCCDWBs8pb4WH4XjeTxPP8PRIJ0HPAJcnjghBCkxBCI3Z5jQLEGH+1s52SiwpJfGkAOAeokSx/EvBpjPFe4H4S7bQQ6JF83moAQ1O6h5UoxvjhLsfTVPZxzIYQqgHfijHOA35J4sQRtUm8lw5LrnMmUK8SdiNV9nWsgZ+r+zOHRLsdBcXDq3baObfwDGBjjHFjjLF/ss1GJ9cp5N+v1fN2Fgwh/GeM8W8xxt+TGCZ9Gonj7/shhIyQGJbaM5U7VsFK+nyFxHe5GcAEEsORS3w/T4fj7bA7e1vyV7WiGOMjIYQM4HX+PeyjPB4DRiR/5XsDWJFcngXcEkLYQWKC8OUxxq0hMUH76RDCZ8BrQGYFxJBSB9B2y4CRITHJdCVwz54rxBg/DIlJ6EuT6yxJPlQHeDIk5gIE4GfJ5VcBj4QQriLR1lVKjPHNEEIeUEDijeLV5EMjganJN+u/Az9ILp+eXP4liV9lvtylrnRuu2eBy0IIS4HlJBKYt0js4zsk2mjnEA9ijNtDYiLlKBJtSYxxbUhMpvzv5JcoSMzx2fma3Fk27doxxrgthHAjiS/R/yCRdJRKCOE/SHywHwPsCCH8lMQwwn/tsWpav9fFGN9J/jDxcYxxDbAmhNACWJDosGETcHGM8YMQwvyQmIT/17j3vJ57SRxHi4C5/PvX/B7AuBDCtmRdI2KMa0JiuNsCYA2JoU0ZKd3RQ8R+jtkM4OHkl6kA/CHGuCEkJvf/d0gMd30ZWF0ZcadCjHFOSccaid5ZP1f3Icb4bAghB1gcQvgaeAa4Lvnw5yFxautjgEv2UcUk4P4QwnUkjsOdfhpC6Emi/d8F/gp8TWJkwd9IvPe9TNVR0udrdxJzjLskP0/PCyH8gMS8vN3ez/esrCoebyHGKtlLp0NUCKEp8D8xMfFNkiSVg5+rZRMSZ2G7OiZOzS8dtsPbJEmSJB0m7OmRJEmSlNbs6ZEkSZKU1kx6JEmSJKU1kx5JkiRJac2kR5JUZYUQLgv/vsCnJEkl8kQGkiRJktKaPT2SpENOCGFECGFpCOGtEMJDIYSTQghzk8vmhhC+nVzvhhDC1cnbL4UQfh9CWBRCWBFC6Fq5eyFJOlSY9EiSDikhhFbAr4BeMcbWJK7kPQX4c4wxG5gB3LGP4tVjjB2AnwITD0K4kqQqwKRHknSo6QXMijF+BhBjXA90Bh5JPv4QcMY+yj6e/J8PNE1hjJKkKsSkR5J0qAnAN0043dfjXyX/bweqV1hEkqQqzaRHknSomQucH0I4DiCEUB94Hfh+8vFhwGuVFJskqQryVzBJ0iElxvhOCOFm4OUQwnZgCTAWeCCEMA5YC/ygMmOUJFUtnrJakiRJUlpzeJskSZKktGbSI0mSJCmtmfRIkiRJSmsmPZIkSZLSmkmPJEmSpLRm0iNJkiQprZn0SJIkSUprJj2SJEmS0tr/A//bqiRByDH9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(14, 6))\n",
    "coins_low = [\"xmr-usd\", \"xrp-usd\", \"dot-usd\", \"aave-usd\", \"uni1-usd\", \"vet-usd\", \"fil-usd\", \"rep-usd\", \"icp-usd\", \"axs-usd\"]\n",
    "coins_high = [x for x in res.coin.unique() if x != \"btc-usd\" and x not in coins_low]\n",
    "coins_btc = [\"btc-usd\"]\n",
    "sns.barplot(data=res[res.coin.isin(coins_low)], x=\"coin\", y=\"metric\", hue=\"model_type\", ax=ax, palette=\"pastel\")\n",
    "plt.show(ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "hourly-conservative",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0MAAAFzCAYAAAD8JdJrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABEX0lEQVR4nO39eXhV5b3//z/fBAQVRFTwg2AFW1SmEGYQERAZWkWF1jqVoZZKnTtoUVsFrZ62v3qsAyqOYBUPKIp6jlZRpA7IAUEiVRHRiojyExShooUSuL9/ZJMTJGAYdkLYz8d15cre917rXu+9VjbhlXute0VKCUmSJEnKNdUquwBJkiRJqgyGIUmSJEk5yTAkSZIkKScZhiRJkiTlJMOQJEmSpJxkGJIkSZKUk6pXdgE746CDDkpNmjSp7DIkSZIk7cbmzp37aUqp/tfbq3QYatKkCXPmzKnsMiRJkiTtxiLig7LaPU1OkiRJUk4yDEmSJEnKSYYhSZIkSTmpSl8zJEmSpP+zfv16li5dytq1ayu7FKlS1KpVi8aNG1OjRo1yLW8YkiRJ2kMsXbqUOnXq0KRJEyKissuRKlRKic8++4ylS5fStGnTcq3jaXKSJEl7iLVr13LggQcahJSTIoIDDzxwu0ZGDUOSJEl7EIOQctn2/vwbhiRJkiTlJMOQJEmSVA5NmjTh008/3eFlVq1axW233ZaN0rSDDEOSJElSBTAM7X6yHoYiIi8i5kXE/2SeHxARz0bEosz3eqWWvTwi3o2IhRHRL9u1SZIkac+2ePFijjrqKIYPH06rVq0466yzeO655+jWrRvNmjVj9uzZrFy5klNOOYX8/Hy6dOnC/PnzAfjss8/o27cvbdu2ZcSIEaSUSvp94IEH6NSpEwUFBYwYMYINGzZ8Yy2XXXYZ7733HgUFBVx66aUMHjyYxx9/vOT1s846iyeeeILx48dz8skn079/f4488kiuvvrqndqutq4iRoYuBhaUen4ZMC2l1AyYlnlORLQATgdaAv2B2yIirwLqkyRJ0h7s3Xff5eKLL2b+/Pm8/fbbPPjgg7z88stcf/31/Md//AejRo2ibdu2zJ8/n//4j/9gyJAhAFx99dUcc8wxzJs3j5NOOoklS5YAsGDBAiZNmsSMGTMoLCwkLy+PCRMmfGMdf/jDH/j2t79NYWEhf/rTnxg+fDjjxo0DYPXq1bzyyit873vfA2D27NlMmDCBwsJCHn74YebMmbPD29XWZfU+QxHRGDgBuA74Zab5ZKBn5vF9wN+AkZn2iSmldcD7EfEu0AmYmc0aJUmStGdr2rQprVu3BqBly5b07t2biKB169YsXryYDz74gEceeQSA4447js8++4zVq1fz4osv8uijjwJwwgknUK9e8QlN06ZNY+7cuXTs2BGAf/3rXzRo0GC76+rRowfnn38+y5cv59FHH+X73/8+1asX//e8T58+HHjggQAMGjSIl19+merVq++S7er/ZPumqzcCvwbqlGo7OKW0DCCltCwiNh3BRsD/llpuaaZtMxFxDnAOwLe+9a0slCxJ0pbWTJ6Utb5r/+C0rPUtCWrWrFnyuFq1aiXPq1WrRlFRUUkAKW3TFM1lTdWcUmLo0KH8/ve/3+naBg8ezIQJE5g4cSL33nvvFtsv/XxXblfFsnaaXEScCCxPKc0t7ypltKUtGlK6M6XUIaXUoX79+jtVoyRJknTssceWnG72t7/9jYMOOoj99ttvs/a//vWvfP755wD07t2byZMns3z5cgBWrlzJBx988I3bqVOnDl988cVmbcOGDePGG28EiketNnn22WdZuXIl//rXv3jsscfo1q3bDm9XW5fNkaFuwEkR8T2gFrBfRDwAfBIRDTOjQg2B5ZnllwKHllq/MfBxFuuTJEmSGD16ND/+8Y/Jz89nn3324b777gNg1KhRnHHGGbRr144ePXqUnJXUokULrr32Wvr27cvGjRupUaMGt956K4cddtg2t3PggQfSrVs3WrVqxXe/+13+9Kc/cfDBB9O8eXNOOeWUzZY95phjGDx4MO+++y5nnnkmHTp0ANih7WrrovSsGFnbSERP4JKU0okR8Sfgs5TSHyLiMuCAlNKvI6Il8CDF1wkdQvHkCs1SSludIqNDhw5pzpw5Wa9fkiRPk1NVsGDBApo3b17ZZWg7fPXVV7Ru3ZrXXnuNunXrAjB+/HjmzJnDmDFjKrm6qqmsz0FEzE0pdfj6spVxn6E/AH0iYhHQJ/OclNKbwEPAW8DTwPnbCkKSJElSVfbcc89x1FFHceGFF5YEIVWsChkZyhZHhiRJFcWRIVUFjgztHj777DN69+69Rfu0adNKZohT9mzPyFC2Z5OTJEmScsqBBx5IYWFhZZehcqiM0+QkSZIkqdIZhiRJkiTlJMOQJEmSpJxkGJIkSZKUk5xAQZIkaQ81efaKXdrfDzrV/8ZlateuzZo1azZrGzt2LPvssw9DhgzZpfV8XZMmTahTpw4RQb169fjLX/6y29yQtKL2wY7adNw+/vhjLrroIiZPnlzZJVUIw5AkSZKy6mc/+1lW+08psel2MdOnT+eggw5i1KhRXHvttdx11127pO9q1XbuhKps74Nd5ZBDDsmZIASeJidJkqQsGz16NNdffz0APXv2ZOTIkXTq1IkjjjiCl156CYANGzZw6aWX0rFjR/Lz87njjjsAWLNmDb1796Zdu3a0bt2axx9/HIDFixfTvHlzzjvvPNq1a8eHH3642Ta7du3KRx99BMCKFSv4/ve/T8eOHenYsSMzZswoae/Tpw/t2rVjxIgRHHbYYXz66adl9v2nP/2ppLZRo0YB8OWXX3LCCSfQpk0bWrVqxaRJxfcju+yyy2jRogX5+flccsklW+yDwsJCunTpQn5+PgMHDuTzzz/f5r4py/jx4znllFMYMGAATZs2ZcyYMdxwww20bduWLl26sHLlSgDee+89+vfvT/v27enevTtvv/02AO+//z5du3alY8eOXHnllSX9Ll68mFatWpU87t69O+3ataNdu3a88sor23nkd3+GIUmSJFWooqIiZs+ezY033sjVV18NwD333EPdunV59dVXefXVV7nrrrt4//33qVWrFlOmTOG1115j+vTp/OpXvyoZBVq4cCFDhgxh3rx5W5wO9/TTT3PKKacAcPHFF/OLX/yCV199lUceeYThw4cDcPXVV3Pcccfx2muvMXDgQJYsWVKyfum+Fy5cyKJFi5g9ezaFhYXMnTuXF198kaeffppDDjmE119/nTfeeIP+/fuzcuVKpkyZwptvvsn8+fP57W9/u8X7HzJkCH/84x+ZP38+rVu3LtkHW9s3W/PGG2/w4IMPMnv2bH7zm9+wzz77MG/ePLp27cpf/vIXAM455xxuueUW5s6dy/XXX895551Xsk/OPfdcXn31Vf7f//t/ZfbfoEEDnn32WV577TUmTZrERRddtM16qiJPk5MkSVKFGjRoEADt27dn8eLFAEydOpX58+eXnKK1evVqFi1aROPGjbniiit48cUXqVatGh999BGffPIJAIcddhhdunTZrO9evXrxySef0KBBA6699loAnnvuOd56662SZf75z3/yxRdf8PLLLzNlyhQA+vfvT7169UqWKd331KlTmTp1Km3btgWKR6sWLVpE9+7dueSSSxg5ciQnnngi3bt3p6ioiFq1ajF8+HBOOOEETjzxxM3qW716NatWraJHjx4ADB06lFNPPXWb+2ZrevXqRZ06dahTpw5169ZlwIABALRu3Zr58+ezZs0aXnnllc36X7duHQAzZszgkUceAWDw4MGMHDlyi/7Xr1/PBRdcQGFhIXl5ebzzzjvbrKcqMgxJkiSpQtWsWROAvLw8ioqKgOJrc2655Rb69eu32bLjx49nxYoVzJ07lxo1atCkSRPWrl0LwL777rtF39OnT2ffffdl2LBhXHXVVdxwww1s3LiRmTNnsvfee2+27KYRprKU7julxOWXX86IESO2WG7u3Lk89dRTXH755fTt25errrqK2bNnM23aNCZOnMiYMWN4/vnny7lnyt4337QsQLVq1UqeV6tWjaKiIjZu3Mj+++9PYWFhmetHxDb7//Of/8zBBx/M66+/zsaNG6lVq1a530dV4WlykiRJqnT9+vXj9ttvZ/369QC88847fPnll6xevZoGDRpQo0YNpk+fzgcffPCNfe29997ceOON/OUvf2HlypX07duXMWPGlLy+KRwcc8wxPPTQQ0Dx6M+ma3fKqu3ee+8tmSXvo48+Yvny5Xz88cfss88+/OhHP+KSSy7htddeY82aNaxevZrvfe973HjjjVsEkbp161KvXr2S64Huv//+klGiXW2//fajadOmPPzww0BxqHv99dcB6NatGxMnTgRgwoQJZa6/evVqGjZsSLVq1bj//vvZsGFDVuqsTI4MSZIk7aHKMxX2rvbVV1/RuHHjkue//OUvy7Xe8OHDWbx4Me3atSOlRP369Xnsscc466yzGDBgAB06dKCgoICjjjqqXP01bNiQM844g1tvvZWbb76Z888/n/z8fIqKijj22GMZO3Yso0aN4owzzmDSpEn06NGDhg0bUqdOnS2mBu/bty8LFiyga9euQPE01A888ADvvvsul156KdWqVaNGjRrcfvvtfPHFF5x88smsXbuWlBJ//vOft6jtvvvu42c/+xlfffUVhx9+OOPGjSvXe9oREyZM4Nxzz+Xaa69l/fr1nH766bRp04abbrqJM888k5tuuonvf//7Za573nnn8f3vf5+HH36YXr16lTkSV9XFtoYHd3cdOnRIc+bMqewyJEk5YM3kSVnru/YPTsta38otCxYsoHnz5pVdRpWxbt068vLyqF69OjNnzuTcc8/d6illqjrK+hxExNyUUoevL+vIkCRJknLSkiVL+OEPf8jGjRvZa6+9dvqeRKp6DEOSJEnKSc2aNWPevHmVXcY2PfPMM1vM9Na0adOSWfC0cwxDkiRJ0m6qX79+W8ywp13H2eQkSZIk5STDkCRJkqScZBiSJEmSlJO8ZkiSJGkPterpW3Zpf/v3v3CX9idVNkeGJEmStMvk5eVRUFBAq1atGDBgAKtWrdol/Y4fP54LLrhgl/TVpEkTWrduTUFBAQUFBbzyyiu7pN+vKyws5Kmnnip5Pn78eOrXr19y89iybsgqGDZsGJMnTwaKb8b71ltvZW1bhiFJkiTtMnvvvTeFhYW88cYbHHDAAdx6662VXVKZpk+fTmFhIYWFhRx99NHlWqeoqGi7tvH1MARw2mmnUVhYyIwZM7juuuv48MMPt6vPXVHXjkopsXHjxgrZ1iZ33303LVq0yFr/hiFJkiRlRdeuXfnoo48AmD17NkcffTRt27bl6KOPZuHChUDxaMmgQYPo378/zZo149e//nXJ+uPGjeOII46gR48ezJgxo6T9gw8+oHfv3uTn59O7d2+WLFkCFI8onHvuufTq1YvDDz+cF154gbPPPpvmzZszbNiwbda6rT5/+ctf0qtXL0aOHMl7771H//79ad++Pd27d+ftt98G4OGHH6ZVq1a0adOGY489ln//+99cddVVTJo0iYKCAiZNmrTZ9g488EC+853vsGzZMgAeeOABOnXqREFBASNGjGDDhg0A3HPPPRxxxBH07NmTn/70pyWjYztaF8Cbb75Zsq38/HwWLVoEwA033ECrVq1o1aoVN954IwCLFy+mefPmnHfeebRr126r4a127dqMHDmS9u3bc/zxxzN79mx69uzJ4YcfzhNPPAHAhg0buPTSS+nYsSP5+fnccccdQHHIuuCCC2jRogUnnHACy5cvL+m3Z8+ezJkzB4Bzzz2XDh060LJlS0aNGrXN41leXjMkSZKkXW7Dhg1MmzaNn/zkJwAcddRRvPjii1SvXp3nnnuOK664gkceeQQoHkGZN28eNWvW5Mgjj+TCCy+kevXqjBo1irlz51K3bl169epF27ZtAbjgggsYMmQIQ4cO5d577+Wiiy7iscceA+Dzzz/n+eef54knnmDAgAHMmDGDu+++m44dO1JYWEhBQQEAvXr1Ii8vj5o1azJr1qxt9vnOO+/w3HPPkZeXR+/evRk7dizNmjVj1qxZnHfeeTz//PNcc801PPPMMzRq1IhVq1ax1157cc011zBnzhzGjBkDFAe/TZYsWcLatWvJz89nwYIFTJo0iRkzZlCjRg3OO+88JkyYwPHHH8/vfvc7XnvtNerUqcNxxx1HmzZtSvrYkboAxo4dy8UXX8xZZ53Fv//9bzZs2MDcuXMZN24cs2bNIqVE586d6dGjB/Xq1WPhwoWMGzeO2267bavH+8svv6Rnz5788Y9/ZODAgfz2t7/l2Wef5a233mLo0KGcdNJJ3HPPPdStW5dXX32VdevW0a1bN/r27cu8efNYuHAhf//73/nkk09o0aIFZ5999hbbuO666zjggAPYsGEDvXv3Zv78+eTn52/3z2ZphiFJkiTtMv/6178oKChg8eLFtG/fnj59+gCwevVqhg4dyqJFi4gI1q9fX7JO7969qVu3LgAtWrTggw8+4NNPP6Vnz57Ur18fKD697J133gFg5syZPProowAMHjx4s9GkAQMGEBG0bt2agw8+mNatWwPQsmVLFi9eXBKGpk+fzkEHHVSy3rb6PPXUU8nLy2PNmjW88sornHrqqSWvrVu3DoBu3boxbNgwfvjDHzJo0KCt7p9JkyYxffp0Fi5cyF133UWtWrWYNm0ac+fOpWPHjiX7sEGDBsyePZsePXpwwAEHlNSxaR/sTF1du3bluuuuY+nSpQwaNIhmzZrx8ssvM3DgQPbdd18ABg0axEsvvcRJJ53EYYcdRpcuXbb6ngD22msv+vfvD0Dr1q2pWbMmNWrUoHXr1ixevBiAqVOnMn/+/JLrgVavXs2iRYt48cUXOeOMM8jLy+OQQw7huOOOK3MbDz30EHfeeSdFRUUsW7aMt956yzAkSZKk3cema4ZWr17NiSeeyK233spFF13ElVdeSa9evZgyZQqLFy+mZ8+eJevUrFmz5HFeXl7JNTARUa5tll5uU1/VqlXbrN9q1apt17U1pfvcFBA2btzI/vvvT2Fh4RbLjx07llmzZvHkk09SUFBQ5jJQHOrGjBnDzJkzOeGEE/jud79LSomhQ4fy+9//frNlp0yZss0ad7SuM888k86dO/Pkk0/Sr18/7r77blJK37idbalRo0bJPiu970vv95QSt9xyC/369dts3aeeeuobj/X777/P9ddfz6uvvkq9evUYNmwYa9eu/ca6volhSJIkaQ9VmVNh161bl5tvvpmTTz6Zc889l9WrV9OoUSNg89PFtqZz585cfPHFfPbZZ+y33348/PDDJaeIHX300UycOJHBgwczYcIEjjnmmJ2utzx97rfffjRt2pSHH36YU089lZQS8+fPp02bNrz33nt07tyZzp0789///d98+OGH1KlThy+++KLM7XXt2pXBgwdz0003MXjwYE4++WR+8Ytf0KBBA1auXMkXX3xBp06d+MUvfsHnn39OnTp1eOSRR0pGunamrtWrV3P44Ydz0UUX8Y9//IP58+dz7LHHMmzYMC677DJSSkyZMoX7779/p/draf369eP222/nuOOOo0aNGrzzzjs0atSIY489ljvuuIMhQ4awfPlypk+fzplnnrnZuv/85z/Zd999qVu3Lp988gl//etfNwvUO8oJFCRJkpQVbdu2pU2bNkycOJFf//rXXH755XTr1q1kcoBtadiwIaNHj6Zr164cf/zxtGvXruS1m2++mXHjxpGfn8/999/PTTfdtNO1lrfPCRMmcM8999CmTRtatmzJ448/DsCll15K69atadWqFcceeyxt2rShV69evPXWW2VOoAAwcuRIxo0bx6GHHsq1115L3759yc/Pp0+fPixbtoxGjRpxxRVX0LlzZ44//nhatGhRcjrhztQ1adIkWrVqRUFBAW+//TZDhgyhXbt2DBs2jE6dOtG5c2eGDx9eco3WrjJ8+HBatGhBu3btaNWqFSNGjKCoqIiBAwfSrFkzWrduzbnnnkuPHj22WLdNmza0bduWli1bcvbZZ9OtW7ddUlNsa0hsd9ehQ4e0aXYJSZKyac3kLf8js6vU/sFpWetbuWXBggU0b968ssvQLrRmzRpq165dEhrOPvtsBg4cWNll7dbK+hxExNyUUoevL5u1kaGIqBURsyPi9Yh4MyKuzrSPjoiPIqIw8/W9UutcHhHvRsTCiOi39d4lSZKkPd/o0aNLbmLbtGlTTjnllMouaY+SzWuG1gHHpZTWREQN4OWI+GvmtT+nlK4vvXBEtABOB1oChwDPRcQRKaVvHkeVJEmS9kDXX3/9Ny9UgTp37lwyU90m999/f5nXMlUFWQtDqfj8uzWZpzUyX9s6J+9kYGJKaR3wfkS8C3QCZmarRkmSJEnlN2vWrMouYZfK6gQKEZEXEYXAcuDZlNKmvXdBRMyPiHsjol6mrRFQ+pa2SzNtX+/znIiYExFzVqxYkc3yJUmSJO3BshqGUkobUkoFQGOgU0S0Am4Hvg0UAMuA/8wsXtbk4luMJKWU7kwpdUgpddh0Ey5JkiRJ2l4VMrV2SmkV8Degf0rpk0xI2gjcRfGpcFA8EnRoqdUaAx9XRH2SJEmSck/WrhmKiPrA+pTSqojYGzge+GNENEwpLcssNhB4I/P4CeDBiLiB4gkUmgGzs1WfJEnSnu6ORRN3aX8jmp2+S/uTKls2Z5NrCNwXEXkUj0A9lFL6n4i4PyIKKD4FbjEwAiCl9GZEPAS8BRQB5zuTnCRJUtVSu3Zt1qxZs1nb2LFj2WeffRgyZEhWt92kSRPq1KlDRFCvXj3+8pe/cNhhh2V1m+VVUftA2yebs8nNB7a4bW1KafA21rkOuC5bNUmSdh+rnr4lK/3u3//CrPQracf97Gc/y2r/KSWKJzKG6dOnc9BBBzFq1CiuvfZa7rrrrl3Sd7VqO3d1Sbb3wfYqKiqievVsjotUDRVyzZAkSZJy1+jRo0vul9OzZ09GjhxJp06dOOKII3jppZcA2LBhA5deeikdO3YkPz+fO+64A4A1a9bQu3dv2rVrR+vWrXn88ccBWLx4Mc2bN+e8886jXbt2fPjhh5tts2vXrnz00UcArFixgu9///t07NiRjh07MmPGjJL2Pn360K5dO0aMGMFhhx3Gp59+Wmbff/rTn0pqGzVqFABffvklJ5xwAm3atKFVq1ZMmjQJgMsuu4wWLVqQn5/PJZdcssU+KCwspEuXLuTn5zNw4EA+//zzbe6bsowfP56TTz6Z/v37c+SRR3L11VeX7JdWrVqVLHf99dczevTokv6vuOIKevTowU033cTcuXPp0aMH7du3p1+/fixbtqysTe3RjIOSJEmqUEVFRcyePZunnnqKq6++mueee4577rmHunXr8uqrr7Ju3Tq6detG3759OfTQQ5kyZQr77bcfn376KV26dOGkk04CYOHChYwbN47bbrtti208/fTTnHLKKQBcfPHF/OIXv+CYY45hyZIl9OvXjwULFnD11Vdz3HHHcfnll/P0009z5513lqxfuu+pU6eyaNEiZs+eTUqJk046iRdffJEVK1ZwyCGH8OSTTwKwevVqVq5cyZQpU3j77beJCFatWrVFbUOGDOGWW26hR48eXHXVVVx99dXceOONW903WzN79mzeeOMN9tlnHzp27MgJJ5zAQQcdtM19v2rVKl544QXWr19Pjx49ePzxx6lfvz6TJk3iN7/5Dffee+8219/TGIYkSZJUoQYNGgRA+/btWbx4MQBTp05l/vz5TJ48GSgOFosWLaJx48ZcccUVvPjii1SrVo2PPvqITz75BIDDDjuMLl26bNZ3r169+OSTT2jQoAHXXnstAM899xxvvfVWyTL//Oc/+eKLL3j55ZeZMmUKAP3796devXoly5Tue+rUqUydOpW2bYuvAFmzZg2LFi2ie/fuXHLJJYwcOZITTzyR7t27U1RURK1atRg+fDgnnHACJ5544mb1rV69mlWrVtGjRw8Ahg4dyqmnnrrNfbM1ffr04cADDyxZ7+WXXy4JgFtz2mmnAcVh74033qBPnz5A8chcw4YNt7nunsgwJEmSpApVs2ZNAPLy8igqKgKKr8255ZZb6Nev32bLjh8/nhUrVjB37lxq1KhBkyZNWLt2LQD77rvvFn1Pnz6dfffdl2HDhnHVVVdxww03sHHjRmbOnMnee++92bKbrjMqS+m+U0pcfvnljBgxYovl5s6dy1NPPcXll19O3759ueqqq5g9ezbTpk1j4sSJjBkzhueff76ce6bsfbM1EbHF8+rVq7Nx48aStk376uvvK6VEy5YtmTlzZrlr2xMZhiRJkvZQVWkq7H79+nH77bdz3HHHUaNGDd555x0aNWrE6tWradCgATVq1GD69Ol88MEH39jX3nvvzY033kjr1q357W9/S9++fRkzZgyXXnopUHzNTkFBAccccwwPPfQQI0eOZOrUqSXX7pRV25VXXslZZ51F7dq1+eijj6hRowZFRUUccMAB/OhHP6J27dqMHz+eNWvW8NVXX/G9732PLl268J3vfGezvurWrUu9evV46aWX6N69O/fff3/JKNH2evbZZ1m5ciV77703jz32GPfeey8HH3wwy5cv57PPPqN27dr8z//8D/37999i3SOPPJIVK1Ywc+ZMunbtyvr163nnnXdo2bLlDtVSVRmGJEmStMt89dVXNG7cuOT5L3/5y3KtN3z4cBYvXky7du1IKVG/fn0ee+wxzjrrLAYMGECHDh0oKCjgqKOOKld/DRs25IwzzuDWW2/l5ptv5vzzzyc/P5+ioiKOPfZYxo4dy6hRozjjjDOYNGkSPXr0oGHDhtSpU2eLqcH79u3LggUL6Nq1K1A8ffgDDzzAu+++y6WXXkq1atWoUaMGt99+O1988QUnn3wya9euJaXEn//85y1qu++++/jZz37GV199xeGHH864cePK9Z6+7phjjmHw4MG8++67nHnmmXTo0AGAq666is6dO9O0adOt7q+99tqLyZMnc9FFF7F69WqKior4+c9/nnNhKLY1PLi769ChQ5ozZ05llyFJ2gFVbWrtNZMnZaVfgNo/OC1rfSu3LFiwgObNm1d2GVXGunXryMvLo3r16sycOZNzzz2XwsLCyi6rXMaPH8+cOXMYM2ZMZZey2ynrcxARc1NKHb6+rCNDkiRJyklLlizhhz/8IRs3bmSvvfba6XsSqeoxDEmSJCknNWvWjHnz5lV2Gdv0zDPPMHLkyM3amjZtypQpUxg2bFjlFLUHMQxJkiRJu6l+/fptMcOedp1qlV2AJEmSJFUGw5AkSZKknGQYkiRJkpSTvGZIkiRpD7Wrp4R3GnjtaRwZkiRJ0i6Tl5dHQUEBrVq1YsCAAaxatWqX9Dt+/HguuOCCXdJXkyZNaN26NQUFBRQUFPDKK6/skn6/rrCwkKeeeqrk+fjx46lfv37JzWPLuiGrKpZhSJIkSbvM3nvvTWFhIW+88QYHHHAAt956a2WXVKbp06dTWFhIYWEhRx99dLnWKSoq2q5tfD0MAZx22mkUFhYyY8YMrrvuOj788MPt6nNX1LWjUkps3Lhxl/dbUfWXxTAkSZKkrOjatSsfffQRALNnz+boo4+mbdu2HH300SxcuBAoHi0ZNGgQ/fv3p1mzZvz6178uWX/cuHEcccQR9OjRgxkzZpS0f/DBB/Tu3Zv8/Hx69+7NkiVLABg2bBjnnnsuvXr14vDDD+eFF17g7LPPpnnz5t94T55t9fnLX/6SXr16MXLkSN577z369+9P+/bt6d69O2+//TYADz/8MK1ataJNmzYce+yx/Pvf/+aqq65i0qRJFBQUMGnS5qcsHnjggXznO99h2bJlADzwwAN06tSJgoICRowYwYYNGwC45557OOKII+jZsyc//elPS0bHdrQugDfffLNkW/n5+SxatAiAG264gVatWtGqVStuvPFGABYvXkzz5s0577zzaNeu3VbDW+3atfnVr35Fu3bt6N27NytWrACgZ8+ezJkzB4BPP/2UJk2alBz3U089lQEDBtC3b1++/PJLzj77bDp27Ejbtm15/PHHt3m8dhWvGZIkSdIut2HDBqZNm8ZPfvITAI466ihefPFFqlevznPPPccVV1zBI488AhSPoMybN4+aNWty5JFHcuGFF1K9enVGjRrF3LlzqVu3Lr169aJt27YAXHDBBQwZMoShQ4dy7733ctFFF/HYY48B8Pnnn/P888/zxBNPMGDAAGbMmMHdd99Nx44dKSwspKCgAIBevXqRl5dHzZo1mTVr1jb7fOedd3juuefIy8ujd+/ejB07lmbNmjFr1izOO+88nn/+ea655hqeeeYZGjVqxKpVq9hrr7245pprmDNnDmPGjAGKA8AmS5YsYe3ateTn57NgwQImTZrEjBkzqFGjBueddx4TJkzg+OOP53e/+x2vvfYaderU4bjjjqNNmzYlfexIXQBjx47l4osv5qyzzuLf//43GzZsYO7cuYwbN45Zs2aRUqJz58706NGDevXqsXDhQsaNG8dtt9221eP95Zdf0q5dO/7zP/+Ta665hquvvrrkfW/NzJkzmT9/PgcccABXXHEFxx13HPfeey+rVq2iU6dOHH/88ey7777l+nnbUYYhSZIk7TL/+te/KCgoYPHixbRv354+ffoAsHr1aoYOHcqiRYuICNavX1+yTu/evalbty4ALVq04IMPPuDTTz+lZ8+e1K9fHyg+veydd94Biv8T/eijjwIwePDgzUaTBgwYQETQunVrDj74YFq3bg1Ay5YtWbx4cUkYmj59OgcddFDJetvq89RTTyUvL481a9bwyiuvcOqpp5a8tm7dOgC6devGsGHD+OEPf8igQYO2un8mTZrE9OnTWbhwIXfddRe1atVi2rRpzJ07l44dO5bswwYNGjB79mx69OjBAQccUFLHpn2wM3V17dqV6667jqVLlzJo0CCaNWvGyy+/zMCBA0vCx6BBg3jppZc46aSTOOyww+jSpctW3xNAtWrVOO204gk2fvSjH21zH2zSp0+fkvc2depUnnjiCa6//noA1q5dy5IlS2jevPk39rMzDEOSJEnaZTZdM7R69WpOPPFEbr31Vi666CKuvPJKevXqxZQpU1i8eDE9e/YsWadmzZolj/Py8kquIYmIcm2z9HKb+qpWrdpm/VarVm27rk0p3eemgLBx40b2339/CgsLt1h+7NixzJo1iyeffJKCgoIyl4HiUDdmzBhmzpzJCSecwHe/+11SSgwdOpTf//73my07ZcqUbda4o3WdeeaZdO7cmSeffJJ+/fpx9913k1L6xu1sj037r3r16iXXGa1du3ar/aaUeOSRRzjyyCO3e1s7w2uGJEmS9lC1f3DaLv3aHnXr1uXmm2/m+uuvZ/369axevZpGjRoBm58utjWdO3fmb3/7G5999hnr16/n4YcfLnnt6KOPZuLEiQBMmDCBY445ZrtqK0t5+txvv/1o2rRpSS0pJV5//XUA3nvvPTp37sw111zDQQcdxIcffkidOnX44osvytxe165dGTx4MDfddBO9e/dm8uTJLF++HICVK1fywQcf0KlTJ1544QU+//xzioqKSk4r3Nm6/vGPf3D44Ydz0UUXcdJJJzF//nyOPfZYHnvsMb766iu+/PJLpkyZQvfu3cu9/zZu3MjkyZMBePDBB0v2X5MmTZg7dy5Ayetl6devH7fccktJKJs3b165t70zDEOSJEnKirZt29KmTRsmTpzIr3/9ay6//HK6detWMjnAtjRs2JDRo0fTtWtXjj/+eNq1a1fy2s0338y4cePIz8/n/vvv56abbtrpWsvb54QJE7jnnnto06YNLVu2LLnQ/9JLL6V169a0atWKY489ljZt2tCrVy/eeuutMidQABg5ciTjxo3j0EMP5dprr6Vv377k5+fTp08fli1bRqNGjbjiiivo3Lkzxx9/PC1atCg5nXBn6po0aRKtWrWioKCAt99+myFDhtCuXTuGDRtGp06d6Ny5M8OHDy+5Rqs89t13X958803at2/P888/z1VXXQXAJZdcwu23387RRx/Np59+utX1r7zyStavX09+fj6tWrXiyiuvLPe2d0Zsa0hsd9ehQ4e0aXYKSVLVsurpW7LS7/79L8xKv7v65pWleSNL7SoLFizI+jUWqlhr1qyhdu3aFBUVMXDgQM4++2wGDhxY2WVtoXbt2qxZs6ayywDK/hxExNyUUoevL+vIkCRJkrSbGj16dMlNbJs2bcopp5xS2SXtUZxAQZIkSdpNbZpdbXfRuXPnkpnqNrn//vt3m1Gh7WUYkiRJ2oOklMo9C5u0vWbNmlXZJWzT9l4C5GlykiRJe4hatWrx2Wefbfd/CKU9QUqJzz77jFq1apV7HUeGJEmS9hCNGzdm6dKlrFixorJLkSpFrVq1aNy4cbmXNwxJkiTtIWrUqEHTpk0ruwypyvA0OUmSJEk5KWthKCJqRcTsiHg9It6MiKsz7QdExLMRsSjzvV6pdS6PiHcjYmFE9MtWbZIkSZKUzZGhdcBxKaU2QAHQPyK6AJcB01JKzYBpmedERAvgdKAl0B+4LSLyslifJEmSpByWtTCUim2acLxG5isBJwP3ZdrvA07JPD4ZmJhSWpdSeh94F+iUrfokSZIk5basXjMUEXkRUQgsB55NKc0CDk4pLQPIfG+QWbwR8GGp1Zdm2r7e5zkRMSci5jhTiiRJkqQdldUwlFLakFIqABoDnSKi1TYWL+vuYFtMkp9SujOl1CGl1KF+/fq7qFJJkiRJuaZCZpNLKa0C/kbxtUCfRERDgMz35ZnFlgKHllqtMfBxRdQnSZIkKfdkcza5+hGxf+bx3sDxwNvAE8DQzGJDgcczj58ATo+ImhHRFGgGzM5WfZIkSZJyWzZvutoQuC8zI1w14KGU0v9ExEzgoYj4CbAEOBUgpfRmRDwEvAUUAeenlDZksT5JkiRJOSxrYSilNB9oW0b7Z0DvraxzHXBdtmqSJEmSpE0q5JohSZIkSdrdGIYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJxmGJEmSJOUkw5AkSZKknGQYkiRJkpSTDEOSJEmScpJhSJIkSVJOMgxJkiRJykmGIUmSJEk5yTAkSZIkKScZhiRJkiTlJMOQJEmSpJxkGJIkSZKUkwxDkiRJknKSYUiSJElSTjIMSZIkScpJhiFJkiRJOckwJEmSJCknGYYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJyUtbCUEQcGhHTI2JBRLwZERdn2kdHxEcRUZj5+l6pdS6PiHcjYmFE9MtWbZIkSZJUPYt9FwG/Sim9FhF1gLkR8WzmtT+nlK4vvXBEtABOB1oChwDPRcQRKaUNWaxRkiRJUo7K2shQSmlZSum1zOMvgAVAo22scjIwMaW0LqX0PvAu0Clb9UmSJEnKbRVyzVBENAHaArMyTRdExPyIuDci6mXaGgEfllptKdsOT5IkSZK0w7IehiKiNvAI8POU0j+B24FvAwXAMuA/Ny1axuqpjP7OiYg5ETFnxYoV2SlakiRJ0h4vq2EoImpQHIQmpJQeBUgpfZJS2pBS2gjcxf+dCrcUOLTU6o2Bj7/eZ0rpzpRSh5RSh/r162ezfEmSJEl7sGzOJhfAPcCClNINpdobllpsIPBG5vETwOkRUTMimgLNgNnZqk+SJElSbsvmbHLdgMHA3yOiMNN2BXBGRBRQfArcYmAEQErpzYh4CHiL4pnozncmOUmSJEnZkrUwlFJ6mbKvA3pqG+tcB1yXrZokSZIkaZMKmU1OkiRJknY3hiFJkiRJOckwJEmSJCknGYYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJxmGJEmSJOWkcoWhiBgYEXVLPd8/Ik7JWlWSJEmSlGXlHRkalVJavelJSmkVMCorFUmSJElSBahezuXKCk3lXVeSJEnaIWsmT8pa37V/cFrW+lbVUN6RoTkRcUNEfDsiDo+IPwNzs1mYJEmSJGVTecPQhcC/gUnAw8Ba4PxsFSVJkiRJ2VauU91SSl8Cl2W5FkmSJEmqMNsMQxFxY0rp5xHx30D6+usppZOyVpkkSZIkZdE3jQzdn/l+fbYLkSRJkqSKtM0wlFKaGxF5wE9TSj+qoJokSZIkKeu+cQKFlNIGoH5E7FUB9UiSJElShSjvvYIWAzMi4gngy02NKaUbslGUJEmSJGVbecPQx5mvakCdTNsWEypIkiRJUlVR3jD0Vkrp4dINEXFqFuqRJEmSpApR3puuXl7ONkmSJEmqEr7pPkPfBb4HNIqIm0u9tB9QlM3CJEmSJCmbvuk0uY+BOcBJwNxS7V8Av8hWUZIkSZKUbd90n6HXgdcj4sHMst9KKS2skMokSZIkKYvKe81Qf6AQeBogIgoy02xLkiRJUpVU3jA0GugErAJIKRUCTba1QkQcGhHTI2JBRLwZERdn2g+IiGcjYlHme71S61weEe9GxMKI6Lf9b0eSJEmSyqe8YagopbR6O/suAn6VUmoOdAHOj4gWwGXAtJRSM2Ba5jmZ104HWlI8EnVbRORt5zYlSZIkqVzKG4beiIgzgbyIaBYRtwCvbGuFlNKylNJrmcdfAAuARsDJwH2Zxe4DTsk8PhmYmFJal1J6H3iX4tEoSZIkSdrlyhuGLqR4xGYd8CCwGri4vBuJiCZAW2AWcHBKaRkUByagQWaxRsCHpVZbmmmTJEmSpF3um6bW3qRF5qt65utkiqfbzv+mFSOiNvAI8POU0j8jYquLltGWyujvHOAcgG9961vlqV2SJEkZq56+JSv97t//wqz0K2VTecPQBOAS4A1gY3k7j4gaFAehCSmlRzPNn0REw5TSsohoCCzPtC8FDi21emOK73O0mZTSncCdAB06dNgiLEmSJElSeZT3NLkVKaX/Tim9n1L6YNPXtlaI4iGge4AFKaUbSr30BDA083go8Hip9tMjomZENAWaAbPL/U4kSZIkaTuUd2RoVETcTfHsb+s2NZYa7SlLN2Aw8PeIKMy0XQH8AXgoIn4CLAFOzfT1ZkQ8BLxF8Ux056eUNmzHe5EkSZKkcitvGPoxcBRQg/87TS4BWw1DKaWXKfs6IIDeW1nnOuC6ctYkSZIkSTusvGGoTUqpdVYrkSRJkqQKVN5rhv43c1NUSZIkSdojlHdk6BhgaES8T/E1QwGklNI3Tq0tSZIkaedla1p0yN2p0csbhvpntQpJkiRJqmDlCkPfNI22JEmSJFU15b1mSJIkSZL2KIYhSZIkSTnJMCRJkiQpJ5V3AgVJkqQqa83kSVnpt/YPTstKv5IqhiNDkiRJknKSI0OSJEmSsmJ3H5V1ZEiSJElSTjIMSZIkScpJhiFJkiRJOckwJEmSJCknGYYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJxmGJEmSJOUkw5AkSZKknGQYkiRJkpSTDEOSJEmScpJhSJIkSVJOMgxJkiRJykmGIUmSJEk5KWthKCLujYjlEfFGqbbREfFRRBRmvr5X6rXLI+LdiFgYEf2yVZckSZIkQXZHhsYD/cto/3NKqSDz9RRARLQATgdaZta5LSLyslibJEmSpByXtTCUUnoRWFnOxU8GJqaU1qWU3gfeBTplqzZJkiRJqoxrhi6IiPmZ0+jqZdoaAR+WWmZppm0LEXFORMyJiDkrVqzIdq2SJEmS9lAVHYZuB74NFADLgP/MtEcZy6ayOkgp3ZlS6pBS6lC/fv2sFClJkiRpz1ehYSil9ElKaUNKaSNwF/93KtxS4NBSizYGPq7I2iRJkiTllgoNQxHRsNTTgcCmmeaeAE6PiJoR0RRoBsyuyNokSZIk5Zbq2eo4Iv4L6AkcFBFLgVFAz4gooPgUuMXACICU0psR8RDwFlAEnJ9S2pCt2iRJkiQpa2EopXRGGc33bGP564DrslWPJEmSJJVWGbPJSZIkSVKlMwxJkiRJyklZO01Ou681kydlre/aPzgta31LkiRJu5IjQ5IkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJzmbnCR9zR2LJmat7xHNTs9a35Ikafs4MiRJkiQpJxmGJEmSJOUkw5AkSZKknGQYkiRJkpSTDEOSJEmScpJhSJIkSVJOMgxJkiRJykmGIUmSJEk5yTAkSZIkKScZhiRJkiTlJMOQJEmSpJxkGJIkSZKUk6pXdgF7glVP35KVfvfvf2FW+pUkaWdk6/ce+LtPUsVyZEiSJElSTjIMSZIkScpJhiFJkiRJOckwJEmSJCknGYYkSZIk5SRnk5OUdZNnr8hKvz/oVD8r/UqSpNzgyJAkSZKknGQYkiRJkpSTshaGIuLeiFgeEW+UajsgIp6NiEWZ7/VKvXZ5RLwbEQsjol+26pIkSZIkyO7I0Hig/9faLgOmpZSaAdMyz4mIFsDpQMvMOrdFRF4Wa5MkSZKU47IWhlJKLwIrv9Z8MnBf5vF9wCml2iemlNallN4H3gU6Zas2SZIkSaroa4YOTiktA8h8b5BpbwR8WGq5pZm2LUTEORExJyLmrFiRnRmqJEmSJO35dpcJFKKMtlTWgimlO1NKHVJKHerXd1pdSZIkSTumosPQJxHRECDzfXmmfSlwaKnlGgMfV3BtkiRJknJIRYehJ4ChmcdDgcdLtZ8eETUjoinQDJhdwbVJkiRJyiHVs9VxRPwX0BM4KCKWAqOAPwAPRcRPgCXAqQAppTcj4iHgLaAIOD+ltCFbtUmSJElS1sJQSumMrbzUeyvLXwdcl616JEmSJKm03WUCBUmSJEmqUIYhSZIkSTnJMCRJkiQpJxmGJEmSJOUkw5AkSZKknGQYkiRJkpSTDEOSJEmScpJhSJIkSVJOMgxJkiRJykmGIUmSJEk5yTAkSZIkKScZhiRJkiTlJMOQJEmSpJxkGJIkSZKUkwxDkiRJknKSYUiSJElSTjIMSZIkScpJhiFJkiRJOckwJEmSJCknGYYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJ1Wv7AIqyuTZK7LW9/FZ61mSJElStjgyJEmSJCknGYYkSZIk5aRKOU0uIhYDXwAbgKKUUoeIOACYBDQBFgM/TCl9Xhn1SZIkSdrzVeY1Q71SSp+Wen4ZMC2l9IeIuCzzfGTllCZJkiTtmGxdq+516rve7jSBwslAz8zj+4C/YRiSpErl5DOSpD1ZZV0zlICpETE3Is7JtB2cUloGkPneoKwVI+KciJgTEXNWrMjeL2lJkiRJe7bKGhnqllL6OCIaAM9GxNvlXTGldCdwJ0CHDh1StgqUJEmStGerlJGhlNLHme/LgSlAJ+CTiGgIkPm+vDJqkyRJkpQbKjwMRcS+EVFn02OgL/AG8AQwNLPYUODxiq5NkiRJUu6ojNPkDgamRMSm7T+YUno6Il4FHoqInwBLgFMroTZJkiRJOaLCw1BK6R9AmzLaPwN6V3Q9kiRJknJTZc0mJ0mSJEmVane6z5AkSTvtjkUTs9LvWURW+pXK4j2+pIphGFJOW/X0LVnre//+F2atb0mSJO08w5BUxayZPCkr/db+wWlZ6VeSJGl35TVDkiRJknKSYUiSJElSTvI0OUmSJO00Jy9RVeTIkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk4yDEmSJEnKSc4mJ0kVyJvmSpK0+zAMSaqyVj19S3Y6/nb97PQrSZJ2K54mJ0mSJCknGYYkSZIk5STDkCRJkqSc5DVDkiRJUo67Y9HErPR7FpGVfncVR4YkSZIk5STDkCRJkqSc5Glyu7FcHa6UJO0ak2evyEq/x2elV0mqeIYhSZK02/APgZIqkmFIVUJV/Oumv9AlSZJ2b14zJEmSJCknGYYkSZIk5STDkCRJkqScZBiSJEmSlJMMQ5IkSZJykmFIkiRJUk7a7cJQRPSPiIUR8W5EXFbZ9UiSJEnaM+1WYSgi8oBbge8CLYAzIqJF5VYlSZIkaU+0W4UhoBPwbkrpHymlfwMTgZMruSZJkiRJe6DdLQw1Aj4s9Xxppk2SJEmSdqlIKVV2DSUi4lSgX0ppeOb5YKBTSunCUsucA5yTeXoksLDCC604BwGfVnYR2mEev6rLY1e1efyqNo9f1eWxq9r29ON3WEqp/tcbq1dGJduwFDi01PPGwMelF0gp3QncWZFFVZaImJNS6lDZdWjHePyqLo9d1ebxq9o8flWXx65qy9Xjt7udJvcq0CwimkbEXsDpwBOVXJMkSZKkPdBuNTKUUiqKiAuAZ4A84N6U0puVXJYkSZKkPdBuFYYAUkpPAU9Vdh27iZw4HXAP5vGrujx2VZvHr2rz+FVdHruqLSeP3241gYIkSZIkVZTd7ZohSZIkSaoQhqFdLCKaRMQb27H86Ii4JMs1DYuIMdncxp4iItZkvh8SEZPLu3wW6/HY7SIR8VRE7J95vMuPm8dq18n252p33XZVVhG/y3aUn83yK/U7sElEnFkZ29auVdE//xExPiJ+UFHb2xUMQ1IZUkofp5Sq1IdZ25ZS+l5KaVVl16EdExF5lV2DlEOaABUahqTKYhjKjuoRcV9EzI+IyRGxT0QsjoirI+K1iPh7RBxVavk2EfF8RCyKiJ+W1eHXk3apv940jIgXI6IwIt6IiO6Z9h9HxDsR8QLQLZtvdk9UeoQv81eVRyPi6cwx+v+VsfxBETEzIk4o4zWPXQWLiJ9l9mthRLwfEdMzn8GDvmE9j9VuJCJ6Zo7dg8DfIyIvIq7P/Bs6PyIu3Mo6/1Pq+ZiIGJZ5/IeIeCuz7vWZtqaZz+6rEfG7inpve4KI+E1ELIyI5yi+CToRURAR/5vZx1Miol6mvWOmbWZE/KnUv695meevZl4fsZVt+dmsWH8Aumf26y/87O2+IuKxiJgbEW9GxDmZtjJ//iNiQETMioh5EfFcRBy8lT5Lfl9GRIeI+FvmcY9Sv1vnRUSdKDYmc3yfBBpk/U3vYoah7DgSuDOllA/8Ezgv0/5pSqkdcDtQ+nSCfOAEoCtwVUQcsh3bOhN4JqVUALQBCiOiIXA1xR+APkCLnXgvKlYAnAa0Bk6LiJKbA2f+MXkSuCql9OR29Omxy5KU0tjMfu1I8c2cb9jJLj1WlacT8JuUUgvgHKAp0Dbz7+uE8nYSEQcAA4GWmXWvzbx0E3B7Sqkj8P/fpZXvwSKiPcX3AmwLDKL4swbwF2BkZh//HRiVaR8H/Cyl1BXYUKqrnwCrM/u/I/DTiGi6HaX42cyOy4CXUkoFKaU/42dvd3Z2Sqk90AG4KCIasfWf/5eBLimltsBE4Nfbua1LgPMzn7fuwL8oPrZHUvz/o58CR+/4W6kchqHs+DClNCPz+AHgmMzjRzPf51I8BL3J4ymlf6WUPgWmU/zLv7xeBX4cEaOB1imlL4DOwN9SSitSSv8GJu3Y21Ap01JKq1NKa4G3gMMy7TWAacCvU0rPbmefHrvsuwl4PqX03zvZj8eq8sxOKb2feXw8MDalVASQUlq5Hf38E1gL3B0Rg4CvMu3dgP/KPL5/F9SbK7oDU1JKX6WU/knxDdL3BfZPKb2QWeY+4NgovlavTkrplUz7g6X66QsMiYhCYBZwINBsO+rws1kx/Oztvi6KiNeB/wUOBQaz9Z//xsAzEfF34FKg5XZuawZwQ0RcRPFnvQg4FvivlNKGlNLHwPM7+X4qnGEoO74+X/mm5+sy3zew+T2etlg+Iq7bNBSZaSsic7wiIoC9AFJKL1L8g/gRcH9EDNlKn9o560o9Ln38iigOt/02veix2z1kTs84jOK/kG1tGY/V7u/LUo+Dr+3ziOhc6rSNkyh1DDNqQfFNvSn+Q9MjwCnA06WW8TjumPLut/iG1y7MjEAUpJSappSm+tnc7fjZ2w1FRE+Kg2rXlFIbYB7wNlvfr7cAY1JKrYERZI5RRDyTOY53Z5YrfSxrbVo5pfQHYDiwN/C/8X+XfFTp42gYyo5vRUTXzOMzKB6W3JaTI6JWRBwI9AReTSn9ZtMvh8wyi4H2m5aneESCiDgMWJ5Sugu4B2hH8V/XekbEgRFRAzh117wtlSEBZwNHRcRlAB67ypc5hecS4EcppY1bW85jVeVMBX4WEdWh+PSblNKsUv+RfgL4AGgRETUjoi7QO7NsbaBu5sbeP6f41Fco/kvn6ZnHZ1XcW6nyXgQGRsTeEVEHGEBxcP08MtftUPwX6hdSSp8DX0REl0z76aX6eQY4N/MZIiKOiIh9/WxWui+AOqWe+9nbPdUFPk8pfZUJJl0oDipb+/mvS/EfDgCGbmpMKfXLHMfhmabF/N/n7fublouIb6eU/p5S+iMwBziK4n8LTo/i68oaAr12+bvMsurfvIh2wAJgaETcASyi+BqhLS42LGU2xdecfAv4XWaY8evuAh6PiNkUn5a16a+lPYFLI2I9sAYYklJaljllYCawDHgNcCamLEkpbYiI04H/joh/ppRu+9oiHruKdwFwADC9+I/IzCnneh6r3dvdwBHA/MyxuAvYbMrYlNKHEfEQMJ/if3/nZV6qQ/GxrUXxX7l/kWm/GHgwIi6m+C/XKoeU0msRMQkopPg/wS9lXhoKjI2IfYB/AD/OtP8EuCsivgT+BqzOtN9N8Wnjr2VGfFZQPHrwdX42K9Z8oChz+tV4ikcU/Oztfp6mOKTOBxZSfKrcMmA0Zf/8jwYejoiPMstu7fq8q4F7IuIKiv+4sMnPI6IXxWfIvAX8Ffg3cBzF1wi+A7xAFRMpVemRLUmStJuLiNoppU0zwF0GNEwpXVzJZUmSI0OSJCnrToiIyyn+f8cHwLDKLUeSijkyJEmSJCknOYGCJEmSpJxkGJIkSZKUkwxDkiRJknKSYUiStMeJiJ+VugGnJEllcgIFSZIkSTnJkSFJUpUREUMiYn5EvB4R90fEYRExLdM2LSK+lVludERcknn8t4j4Y0TMjoh3IqJ75b4LSdLuwjAkSaoSIqIl8BvguJRSG4rvYD8G+EtKKR+YANy8ldWrp5Q6AT8HRlVAuZKkKsAwJEmqKo4DJqeUPgVIKa0EugIPZl6/HzhmK+s+mvk+F2iSxRolSVWIYUiSVFUE8E0Xum7t9XWZ7xuA6rusIklSlWYYkiRVFdOAH0bEgQARcQDwCnB65vWzgJcrqTZJUhXkX8ckSVVCSunNiLgOeCEiNgDzgIuAeyPiUmAF8OPKrFGSVLU4tbYkSZKknORpcpIkSZJykmFIkiRJUk4yDEmSJEnKSYYhSZIkSTnJMCRJkiQpJxmGJEmSJOUkw5AkSZKknGQYkiRJkpST/j/Rk/60BDgWmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(14, 6))\n",
    "sns.barplot(data=res[res.coin.isin(coins_high)], x=\"coin\", y=\"metric\", hue=\"model_type\", ax=ax, palette=\"pastel\")\n",
    "plt.show(ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "crazy-forward",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>coin</th>\n",
       "      <th>metric</th>\n",
       "      <th>type</th>\n",
       "      <th>model_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>70.721581</td>\n",
       "      <td>media</td>\n",
       "      <td>LinearRegression_media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>3669.071687</td>\n",
       "      <td>media</td>\n",
       "      <td>RandomForestRegressor_media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LinearRegression</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>69.574715</td>\n",
       "      <td>pure</td>\n",
       "      <td>LinearRegression_pure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestRegressor</td>\n",
       "      <td>btc-usd</td>\n",
       "      <td>3606.136179</td>\n",
       "      <td>pure</td>\n",
       "      <td>RandomForestRegressor_pure</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   model     coin       metric   type  \\\n",
       "4       LinearRegression  btc-usd    70.721581  media   \n",
       "5  RandomForestRegressor  btc-usd  3669.071687  media   \n",
       "4       LinearRegression  btc-usd    69.574715   pure   \n",
       "5  RandomForestRegressor  btc-usd  3606.136179   pure   \n",
       "\n",
       "                    model_type  \n",
       "4       LinearRegression_media  \n",
       "5  RandomForestRegressor_media  \n",
       "4        LinearRegression_pure  \n",
       "5   RandomForestRegressor_pure  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[res.coin == \"btc-usd\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opened-behavior",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "referenced-champagne",
   "metadata": {},
   "outputs": [],
   "source": [
    "res0.model = res0.model.astype(str).str.encode('utf-8')\n",
    "res0.coin = res0.coin.astype(str).str.encode('utf-8')\n",
    "res0[\"model_coin\"] = res0.model + res0.coin\n",
    "res1.model = res1.model.astype(str).str.encode('utf-8')\n",
    "res1.coin = res1.coin.astype(str).str.encode('utf-8')\n",
    "res1[\"model_coin\"] = res1.model + res1.coin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "effective-concern",
   "metadata": {},
   "outputs": [],
   "source": [
    "res0 = res0.sort_values(\"model_coin\")\n",
    "res1 = res1.sort_values(\"model_coin\")\n",
    "temp = pd.concat((res0, res1.rename(columns={\"metric\": \"metric_pure\"})), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "opposite-reasoning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>model</th>\n",
       "      <th>coin</th>\n",
       "      <th>coin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'aave-usd'</td>\n",
       "      <td>b'aave-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'icp-usd'</td>\n",
       "      <td>b'icp-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'LinearRegression'</td>\n",
       "      <td>b'zil-usd'</td>\n",
       "      <td>b'zil-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'bnb-usd'</td>\n",
       "      <td>b'bnb-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'doge-usd'</td>\n",
       "      <td>b'doge-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'lrc-usd'</td>\n",
       "      <td>b'lrc-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'ltc-usd'</td>\n",
       "      <td>b'ltc-usd'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'RandomForestRegressor'</td>\n",
       "      <td>b'zil-usd'</td>\n",
       "      <td>b'zil-usd'</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       model                     model         coin  \\\n",
       "12       b'LinearRegression'       b'LinearRegression'  b'aave-usd'   \n",
       "28       b'LinearRegression'       b'LinearRegression'   b'icp-usd'   \n",
       "20       b'LinearRegression'       b'LinearRegression'   b'zil-usd'   \n",
       "1   b'RandomForestRegressor'  b'RandomForestRegressor'   b'bnb-usd'   \n",
       "25  b'RandomForestRegressor'  b'RandomForestRegressor'  b'doge-usd'   \n",
       "23  b'RandomForestRegressor'  b'RandomForestRegressor'   b'lrc-usd'   \n",
       "31  b'RandomForestRegressor'  b'RandomForestRegressor'   b'ltc-usd'   \n",
       "21  b'RandomForestRegressor'  b'RandomForestRegressor'   b'zil-usd'   \n",
       "\n",
       "           coin  \n",
       "12  b'aave-usd'  \n",
       "28   b'icp-usd'  \n",
       "20   b'zil-usd'  \n",
       "1    b'bnb-usd'  \n",
       "25  b'doge-usd'  \n",
       "23   b'lrc-usd'  \n",
       "31   b'ltc-usd'  \n",
       "21   b'zil-usd'  "
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp[temp[\"metric\"] < temp[\"metric_pure\"]][[\"model\", \"coin\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "cooked-database",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2222222222222222"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(temp[\"metric\"] < temp[\"metric_pure\"]).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "crude-editing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1506717349034812"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "media_better = temp[temp.metric < temp.metric_pure]\n",
    "((media_better.metric_pure - media_better.metric) / media_better.metric_pure).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "advised-spyware",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}