E1-Random-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d1963a61-9587-45f7-a5d4-df791d6a64a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from nasbench import api\n",
    "from search_spaces import load_nasbench_101\n",
    "from random_search import run_random_search, random_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "367ebcec-78ec-4141-9f3a-c9ef8c86a812",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "os.environ[\"TF_FORCE_GPU_ALLOW_GROWTH\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "051fd7ec-2040-46ee-9444-f66d255e981b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset from file... This may take a few minutes...\n",
      "WARNING:tensorflow:From /data2/home/patara/CS470-Project-Team3/nasbench/api.py:146: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n",
      "Loaded dataset in 46 seconds\n"
     ]
    }
   ],
   "source": [
    "nasbench = load_nasbench_101()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dabdae97-abb1-49c2-bbf0-3a45c8cd154a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change the experiment setting here\n",
    "# 600 runs, max 5000 samples, time limit set to 8e6 according to the paper\n",
    "\n",
    "LOOPS = 60\n",
    "MAX_SAMPLES = 500\n",
    "MAX_TIME_BUDGET = 8e5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd92d7c4-e548-489d-a203-b5495d29865a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c115598f737b42c999f1660c7fe601af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "random_val_avg, random_test_avg = [], []\n",
    "random_val_std, random_test_std = [], []\n",
    "\n",
    "val_acc, test_acc = [], []\n",
    "for _ in tqdm(range(MAX_SAMPLES)):\n",
    "    loop_val, loop_test = [], []\n",
    "    for _ in range(LOOPS):\n",
    "        metric = nasbench.query(random_spec(nasbench))\n",
    "        loop_val.append(metric['validation_accuracy'] * 100)\n",
    "        loop_test.append(metric['test_accuracy'] * 100)\n",
    "\n",
    "    val_acc.append(np.mean(loop_val))\n",
    "    test_acc.append(np.mean(loop_test))\n",
    "    \n",
    "    random_val_avg.append(np.max(val_acc))\n",
    "    random_val_std.append(np.std(loop_val))\n",
    "    \n",
    "    random_test_avg.append(np.max(test_acc))\n",
    "    random_test_std.append(np.std(loop_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "06526ef2-bb22-4442-b533-786e264b7431",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('outputs/random_val_avg_by_samples.npy', random_val_avg)\n",
    "np.save('outputs/random_val_std_by_samples.npy', random_val_std)\n",
    "np.save('outputs/random_test_avg_by_samples.npy', random_test_avg)\n",
    "np.save('outputs/random_test_std_by_samples.npy', random_test_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c253b998-3cc0-4961-9856-3e794c991b4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09ff6e0128214950b247557efa228baa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "random_val_avg, random_test_avg = [], []\n",
    "random_val_std, random_test_std = [], []\n",
    "\n",
    "val_acc, test_acc = [], []\n",
    "for _ in tqdm(range(2000)):\n",
    "    metric = nasbench.query(random_spec(nasbench))\n",
    "    val_acc.append(metric['validation_accuracy'] * 100)\n",
    "    test_acc.append(metric['test_accuracy'] * 100)\n",
    "\n",
    "    \n",
    "    random_val_avg.append(np.max(val_acc))    \n",
    "    random_test_avg.append(np.max(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "de3bed40-92a2-4efe-a527-8f8a18f5dc74",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('outputs/random_val_avg_2000.npy', random_val_avg)\n",
    "np.save('outputs/random_test_avg_2000.npy', random_test_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15088a11-3fcf-4dd2-a2a1-cd044f09ad26",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}