launch_track_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T15:24:12.196261Z",
     "start_time": "2024-06-05T15:24:11.937119Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fish-gcp005\n",
      "/home/akarshkumar0101/nca-alife\n",
      "/home/akarshkumar0101/miniconda3/envs/nca-alife-jax/bin/python\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "!hostname\n",
    "!pwd\n",
    "import os, sys\n",
    "print(sys.executable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b14fab765a9b502f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T15:24:46.791547Z",
     "start_time": "2024-06-05T15:24:46.534227Z"
    }
   },
   "outputs": [],
   "source": [
    "import os, sys, glob, pickle\n",
    "from functools import partial\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xarray as xr\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "# sns.set_theme()\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "from einops import rearrange, reduce, repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd70dd980a69d0d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T16:32:40.437767Z",
     "start_time": "2024-06-05T16:32:40.418237Z"
    }
   },
   "outputs": [],
   "source": [
    "# import experiment_utils\n",
    "import time\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d25a8459feec83fd",
   "metadata": {},
   "source": [
    "# Launch Experiment (on this node)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8c39a5bfd56aad84",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T16:32:45.724746Z",
     "start_time": "2024-06-05T16:32:45.705123Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./src/experiments/main.sh has 6 commands: \n",
      "python main_illuminate_lenia.py  --k_nbrs=2 --n_iters=10000 --save_dir=\"/home/akarshkumar0101/nca-alife-data/illumination/lenia\"\n",
      "python main_illuminate_plenia.py --k_nbrs=2 --n_iters=120000  --save_dir=\"/home/akarshkumar0101/nca-alife-data/illumination/plenia\"\n",
      "python main_illuminate_nca.py    --k_nbrs=2 --n_iters=12000  --save_dir=\"/home/akarshkumar0101/nca-alife-data/illumination/nca\" \n",
      "...\n",
      "Logging to /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03\n"
     ]
    }
   ],
   "source": [
    "header = '\\n'.join([\n",
    "    '#!/bin/bash',\n",
    "    \n",
    "    # 'source ~/.virtualenvs/nca-alife-torch/bin/activate'\n",
    "    \n",
    "    'source ~/miniconda3/etc/profile.d/conda.sh',\n",
    "    'conda activate nca-alife-jax',\n",
    "    \n",
    "    'cd ~/nca-alife/src'\n",
    "])\n",
    "\n",
    "\n",
    "gpus = [2, 3, 4, 5, 6, 7]\n",
    "log_dir = os.path.expanduser(f'~/nca-alife-data/logs/{time.strftime(\"%Y_%m_%d_%H_%M_%S\")}')\n",
    "exp_sh = f'./src/experiments/main.sh'\n",
    "with open(exp_sh, 'r') as f:\n",
    "    commands = [c for c in f.read().split('\\n') if c]\n",
    "print(f\"{exp_sh} has {len(commands)} commands: \")\n",
    "print('\\n'.join(commands[:3]), '\\n...')\n",
    "print(f\"Logging to {log_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "606a1e32-67e7-4c6c-b426-614ccd1651c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03\n"
     ]
    }
   ],
   "source": [
    "print(log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8483676c-0242-42c2-9823-79fad500a436",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9819a394-2848-4e7a-b553-6d07fb1c8c80",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a6eb46d-b180-47c0-9e47-045a0f8f9dde",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ccbc7bd83eb023cb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T16:32:47.309708Z",
     "start_time": "2024-06-05T16:32:47.171678Z"
    }
   },
   "outputs": [],
   "source": [
    "!mkdir -p {log_dir}\n",
    "!rm {os.path.dirname(log_dir)+'/latest'}\n",
    "!ln -sf {log_dir} {os.path.dirname(log_dir)+'/latest'}\n",
    "for i_gpu, gpu in enumerate(gpus):\n",
    "    job_ids = np.arange(len(commands))[i_gpu::len(gpus)]\n",
    "    cvd = f'export CUDA_VISIBLE_DEVICES={gpu}'\n",
    "    txt = f\"{header}\\n{cvd}\\n\\n\"\n",
    "    for job_id in job_ids:\n",
    "        txt += f\"touch {log_dir}/job_{job_id:05d}.status\\n\"\n",
    "        txt += f\"{commands[job_id]} &> {log_dir}/job_{job_id:05d}.out\\n\"\n",
    "        txt += f\"echo $? > {log_dir}/job_{job_id:05d}.status\\n\\n\"\n",
    "    with open(f\"{log_dir}/gpu_{gpu}.sh\", \"w\") as f:\n",
    "        f.write(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fd11b06be751b124",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T16:32:47.659970Z",
     "start_time": "2024-06-05T16:32:47.641164Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#!/bin/bash\n",
      "source ~/miniconda3/etc/profile.d/conda.sh\n",
      "conda activate nca-alife-jax\n",
      "cd ~/nca-alife/src\n",
      "export CUDA_VISIBLE_DEVICES=7\n",
      "\n",
      "touch /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/job_00005.status\n",
      "python main_illuminate_plife.py  --k_nbrs=2 --n_iters=800    --save_dir=\"/home/akarshkumar0101/nca-alife-data/illumination/plife\" &> /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/job_00005.out\n",
      "echo $? > /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/job_00005.status\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('\\n'.join(txt.split('\\n')[:24]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7a1ba5-f8cf-4cb2-93ba-25a548a0f499",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e48a7c8-f06d-4b33-9226-b0800279667e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e5ce96252f75f2f9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-05T16:35:49.567207Z",
     "start_time": "2024-06-05T16:35:49.546023Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_2.sh\n",
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_3.sh\n",
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_4.sh\n",
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_5.sh\n",
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_6.sh\n",
      "bash /home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03/gpu_7.sh\n"
     ]
    }
   ],
   "source": [
    "for i_gpu, gpu in enumerate(gpus):\n",
    "    command = f\"bash {log_dir}/gpu_{gpu}.sh\"\n",
    "    print(command)\n",
    "    process = subprocess.Popen(command, shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d4b43ea-3cd2-4567-9e7b-819a109f4672",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0993c800-10a3-4d44-a37b-55aff7b6da13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f6328aa-5b2e-4739-b80d-2ebc1761740a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "233f54f7c2310a69",
   "metadata": {},
   "source": [
    "# Track Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "323b77b2-6677-4b3b-9864-10f98df4c77d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "a = subprocess.run(['ps', 'ux'], stdout=subprocess.PIPE).stdout.decode('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6919b53c-0578-4e33-a86b-471b78449bee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['akarshk+ 1271851  0.0  0.0   2480   572 ?        S    04:58   0:00 /bin/sh -c bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_4.sh',\n",
       " 'akarshk+ 1271852  0.0  0.0   2480   516 ?        S    04:58   0:00 /bin/sh -c bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_5.sh',\n",
       " 'akarshk+ 1271853  0.0  0.0 219972  3408 ?        S    04:58   0:00 bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_4.sh',\n",
       " 'akarshk+ 1271854  0.0  0.0   2480   512 ?        S    04:58   0:00 /bin/sh -c bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_6.sh',\n",
       " 'akarshk+ 1271855  0.0  0.0 219972  3200 ?        S    04:58   0:00 bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_5.sh',\n",
       " 'akarshk+ 1271856  0.0  0.0   2480   516 ?        S    04:58   0:00 /bin/sh -c bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_7.sh',\n",
       " 'akarshk+ 1271857  0.0  0.0 219972  3460 ?        S    04:58   0:00 bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_6.sh',\n",
       " 'akarshk+ 1271858  0.0  0.0 219972  3188 ?        S    04:58   0:00 bash /home/akarshkumar0101/nca-alife-data/logs/2024_07_31_04_58_39/gpu_7.sh']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Terminated\n",
      "Terminated\n",
      "Terminated\n",
      "Terminated\n"
     ]
    }
   ],
   "source": [
    "[l for l in a.split('\\n') if \"bash\" in l and \"gpu\" in l]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b02e737-1457-43f7-87d9-51b427af547e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "852dbcf0-5257-48ab-ab62-b1a5b15b313d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "577c510a-4b86-40ea-abbe-f3bcccd0dc38",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1eb72d2c-d077-466b-b7a3-c70eb8666291",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoT0lEQVR4nO3dfVjUdb7/8degObAmYypyU5SaFqaGLSlh1mpSyNlj4bqucdwj3natRzsZeROdvCnbi2628uzqwa3rIHmZR3NPYqXLSckwE3XF2M22vNRF0UsG0WRGcB0I5vfHXs7+ZgVSHJgZPs/HdX2vq+/tvL/9w/Oa+c5ocbvdbgEAABgkxN8DAAAAtDcCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADG6ezvAQJRY2OjTp8+rW7duslisfh7HAAAcBXcbrcuXLigmJgYhYS0/B4PAdSE06dPKzY21t9jAACAVjh58qRuueWWFo8hgJrQrVs3SX/7HxgeHu7naQAAwNVwOp2KjY31/B1vCQHUhMsfe4WHhxNAAAAEmat5fIWHoAEAgHEIIAAAYBwCCAAAGIcAAtDhZGdna9iwYerWrZt69+6ttLQ0HT582OuYS5cuac6cOerZs6duvPFGTZgwQZWVlX6aGEB7I4AAdDhFRUWaM2eO9u7dq+3bt6u+vl6PPPKIamtrPcc8/fTT+vDDD7Vp0yYVFRXp9OnT+slPfuLHqQG0J4vb7Xb7e4hA43Q6ZbPZ5HA4+BYY0AFUVVWpd+/eKioq0oMPPiiHw6GIiAitX79eP/3pTyVJ33zzjQYOHKji4mLdd999fp4YQGtcy99v3gEC0OE5HA5JUo8ePSRJJSUlqq+vV3JysueYuLg43XrrrSouLvbLjADaFwEEoENrbGzUvHnzdP/992vw4MGSJLvdri5duqh79+5ex0ZGRsput/thSgDtjR9CBNChzZkzR4cOHdLu3bv9PQqAAMI7QAA6rLlz5+qjjz7Szp07vf5doKioKNXV1am6utrr+MrKSkVFRbXzlAD8gQAC0OG43W7NnTtXmzdv1ieffKK+fft67U9ISNANN9ygwsJCz7bDhw+rvLxcSUlJ7T0uAD/wawC11W91uN1uLVmyRNHR0QoLC1NycrKOHDnSlrcCIIDMmTNH69at0/r169WtWzfZ7XbZ7Xb99a9/lSTZbDbNmDFDmZmZ2rlzp0pKSjRt2jQlJSXxDTDAEH4NoLb6rY5XX31Vv/71r7V69Wrt27dPXbt2VUpKii5dutTWtwQgAOTk5MjhcGjUqFGKjo72LBs3bvQc8+abb+qf//mfNWHCBD344IOKiorS+++/78epAbSngPodIF/8Vofb7VZMTIyeeeYZzZ8/X9LfvgIbGRmpvLw8Pf744987B78DBABA8Ana3wHyxW91lJWVyW63e51js9mUmJjY7Dkul0tOp9NrAQAAHVfAfA3eV7/VcXl7ZGTkVZ+TnZ2tF1544TrvIDAlLFjr7xEAAEGg5LUp/h6hXQXMO0CXf6tjw4YN7f7aWVlZcjgcnuXkyZPtPgMAAGg/ARFAvvytjsvb//GbYi2dY7VaFR4e7rUAAICOy68B1Ba/1dG3b19FRUV5neN0OrVv3z5+3wMAAEjycwD56rc64uLitHnzZkmSxWLRvHnz9NJLL+mDDz7Ql19+qSlTpigmJkZpaWn+uE0AABBg/PoQdE5OjiRp1KhRXtvXrFmjqVOnSvrbb3WEhIRowoQJcrlcSklJ0X/91395HX/48GHPN8gkaeHChaqtrdUTTzyh6upqjRw5UgUFBQoNDW3T+wEAAMEhoH4HKFB0pN8B4ltgAICr0RG+BRa0vwMEAADQHgggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHH8GkC7du3SuHHjFBMTI4vFovz8fK/9FoulyeW1115r9prLli274vi4uLg2vhMAABBM/BpAtbW1io+P16pVq5rcX1FR4bXk5ubKYrFowoQJLV530KBBXuft3r27LcYHAABBqrM/Xzw1NVWpqanN7o+KivJa37Jli0aPHq1+/fq1eN3OnTtfcS4AAMBlQfMMUGVlpbZu3aoZM2Z877FHjhxRTEyM+vXrp8mTJ6u8vLzF410ul5xOp9cCAAA6rqAJoHfeeUfdunXTT37ykxaPS0xMVF5engoKCpSTk6OysjI98MADunDhQrPnZGdny2azeZbY2Fhfjw8AAAJI0ARQbm6uJk+erNDQ0BaPS01N1cSJE3X33XcrJSVF27ZtU3V1td57771mz8nKypLD4fAsJ0+e9PX4AAAggPj1GaCr9dlnn+nw4cPauHHjNZ/bvXt33XHHHTp69Gizx1itVlmt1usZEQAABJGgeAfov//7v5WQkKD4+PhrPrempkbHjh1TdHR0G0wGAACCkV8DqKamRqWlpSotLZUklZWVqbS01OuhZafTqU2bNmnmzJlNXmPMmDFauXKlZ33+/PkqKirS8ePHtWfPHo0fP16dOnVSenp6m94LAAAIHn79COzAgQMaPXq0Zz0zM1OSlJGRoby8PEnShg0b5Ha7mw2YY8eO6ezZs571U6dOKT09XefOnVNERIRGjhypvXv3KiIiou1uBAAABBWL2+12+3uIQON0OmWz2eRwOBQeHu7vca5LwoK1/h4BABAESl6b4u8Rrtu1/P0OimeAAAAAfIkAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABjHrwG0a9cujRs3TjExMbJYLMrPz/faP3XqVFksFq9l7Nix33vdVatWqU+fPgoNDVViYqL279/fRncAAACCkV8DqLa2VvHx8Vq1alWzx4wdO1YVFRWe5X/+539avObGjRuVmZmppUuX6uDBg4qPj1dKSorOnDnj6/EBAECQ6uzPF09NTVVqamqLx1itVkVFRV31Nd944w3NmjVL06ZNkyStXr1aW7duVW5urp599tnrmhcAAHQMAf8M0KeffqrevXvrzjvv1OzZs3Xu3Llmj62rq1NJSYmSk5M920JCQpScnKzi4uJmz3O5XHI6nV4LAADouAI6gMaOHau1a9eqsLBQr7zyioqKipSamqqGhoYmjz979qwaGhoUGRnptT0yMlJ2u73Z18nOzpbNZvMssbGxPr0PAAAQWPz6Edj3efzxxz3/PWTIEN199926/fbb9emnn2rMmDE+e52srCxlZmZ61p1OJxEEAEAHFtDvAP2jfv36qVevXjp69GiT+3v16qVOnTqpsrLSa3tlZWWLzxFZrVaFh4d7LQAAoOMKqgA6deqUzp07p+jo6Cb3d+nSRQkJCSosLPRsa2xsVGFhoZKSktprTAAAEOD8GkA1NTUqLS1VaWmpJKmsrEylpaUqLy9XTU2NFixYoL179+r48eMqLCzUY489pv79+yslJcVzjTFjxmjlypWe9czMTL399tt655139PXXX2v27Nmqra31fCsMAADAr88AHThwQKNHj/asX34OJyMjQzk5OfrTn/6kd955R9XV1YqJidEjjzyi5cuXy2q1es45duyYzp4961mfNGmSqqqqtGTJEtntdg0dOlQFBQVXPBgNAADMZXG73W5/DxFonE6nbDabHA5H0D8PlLBgrb9HAAAEgZLXpvh7hOt2LX+/g+oZIAAAAF8ggAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADG8WsA7dq1S+PGjVNMTIwsFovy8/M9++rr67Vo0SINGTJEXbt2VUxMjKZMmaLTp0+3eM1ly5bJYrF4LXFxcW18JwAAIJj4NYBqa2sVHx+vVatWXbHv4sWLOnjwoBYvXqyDBw/q/fff1+HDh/Xoo49+73UHDRqkiooKz7J79+62GB8AAASpzv588dTUVKWmpja5z2azafv27V7bVq5cqeHDh6u8vFy33nprs9ft3LmzoqKifDorAADoOILqGSCHwyGLxaLu3bu3eNyRI0cUExOjfv36afLkySovL2/xeJfLJafT6bUAAICOK2gC6NKlS1q0aJHS09MVHh7e7HGJiYnKy8tTQUGBcnJyVFZWpgceeEAXLlxo9pzs7GzZbDbPEhsb2xa3AAAAAkRQBFB9fb1+9rOfye12Kycnp8VjU1NTNXHiRN19991KSUnRtm3bVF1drffee6/Zc7KysuRwODzLyZMnfX0LAAAggPj1GaCrcTl+Tpw4oU8++aTFd3+a0r17d91xxx06evRos8dYrVZZrdbrHRUAAASJgH4H6HL8HDlyRDt27FDPnj2v+Ro1NTU6duyYoqOj22BCAAAQjPwaQDU1NSotLVVpaakkqaysTKWlpSovL1d9fb1++tOf6sCBA3r33XfV0NAgu90uu92uuro6zzXGjBmjlStXetbnz5+voqIiHT9+XHv27NH48ePVqVMnpaent/ftAQCAAOXXj8AOHDig0aNHe9YzMzMlSRkZGVq2bJk++OADSdLQoUO9ztu5c6dGjRolSTp27JjOnj3r2Xfq1Cmlp6fr3LlzioiI0MiRI7V3715FRES07c0AAICg4dcAGjVqlNxud7P7W9p32fHjx73WN2zYcL1jAQCADi6gnwECAABoCwQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOO0KoBOnjypU6dOedb379+vefPm6a233vLZYAAAAG2lVQH0L//yL9q5c6ckyW636+GHH9b+/fv1H//xH3rxxRd9OiAAAICvtSqADh06pOHDh0uS3nvvPQ0ePFh79uzRu+++q7y8PF/OBwAA4HOtCqD6+npZrVZJ0o4dO/Too49KkuLi4lRRUeG76QAAANpAqwJo0KBBWr16tT777DNt375dY8eOlSSdPn1aPXv29OmAAAAAvtaqAHrllVf029/+VqNGjVJ6erri4+MlSR988IHnozEAAIBA1bk1J40aNUpnz56V0+nUTTfd5Nn+xBNPqGvXrj4bDgAAoC206h2ghx56SBcuXPCKH0nq0aOHJk2a5JPBAAAA2kqrAujTTz9VXV3dFdsvXbqkzz777LqHAgAAaEvX9BHYn/70J89///nPf5bdbvesNzQ0qKCgQDfffLPvpgMAAGgD1xRAQ4cOlcVikcVi0UMPPXTF/rCwMP3mN7/x2XAAAABt4ZoCqKysTG63W/369dP+/fsVERHh2delSxf17t1bnTp18vmQAAAAvnRNAXTbbbdJkhobG9tkGAAAgPbQqq/BS9KRI0e0c+dOnTlz5oogWrJkyXUPBgAA0FZaFUBvv/22Zs+erV69eikqKkoWi8Wzz2KxEEAAACCgtSqAXnrpJf3yl7/UokWLfD0PAABAm2vV7wCdP39eEydO9PUsAAAA7aJVATRx4kR9/PHHvp4FAACgXbTqI7D+/ftr8eLF2rt3r4YMGaIbbrjBa/+///u/+2Q4AACAttCqAHrrrbd04403qqioSEVFRV77LBYLAQQAAAJaqwKorKzM13MAAAC0m1Y9AwQAABDMWvUO0PTp01vcn5ub26phAAAA2kOrAuj8+fNe6/X19Tp06JCqq6ub/EdSAQAAAkmrAmjz5s1XbGtsbNTs2bN1++23X/dQAAAAbclnzwCFhIQoMzNTb7755lWfs2vXLo0bN04xMTGyWCzKz8/32u92u7VkyRJFR0crLCxMycnJOnLkyPded9WqVerTp49CQ0OVmJio/fv3X+vtAACADsynD0EfO3ZM33333VUfX1tbq/j4eK1atarJ/a+++qp+/etfa/Xq1dq3b5+6du2qlJQUXbp0qdlrbty4UZmZmVq6dKkOHjyo+Ph4paSk6MyZM9d8PwAAoGNq1UdgmZmZXutut1sVFRXaunWrMjIyrvo6qampSk1NbXKf2+3WihUr9Pzzz+uxxx6TJK1du1aRkZHKz8/X448/3uR5b7zxhmbNmqVp06ZJklavXq2tW7cqNzdXzz777FXPBgAAOq5WBdAXX3zhtR4SEqKIiAi9/vrr3/sNsatVVlYmu92u5ORkzzabzabExEQVFxc3GUB1dXUqKSlRVlaW12zJyckqLi5u9rVcLpdcLpdn3el0+uQeAABAYGpVAO3cudPXc1zBbrdLkiIjI722R0ZGevb9o7Nnz6qhoaHJc7755ptmXys7O1svvPDCdU4MAACCxXU9A1RVVaXdu3dr9+7dqqqq8tVM7S4rK0sOh8OznDx50t8jAQCANtSqAKqtrdX06dMVHR2tBx98UA8++KBiYmI0Y8YMXbx40SeDRUVFSZIqKyu9tldWVnr2/aNevXqpU6dO13SOJFmtVoWHh3stAACg42pVAGVmZqqoqEgffvihqqurVV1drS1btqioqEjPPPOMTwbr27evoqKiVFhY6NnmdDq1b98+JSUlNXlOly5dlJCQ4HVOY2OjCgsLmz0HAACYp1XPAP3v//6vfve732nUqFGebf/0T/+ksLAw/exnP1NOTs5VXaempkZHjx71rJeVlam0tFQ9evTQrbfeqnnz5umll17SgAED1LdvXy1evFgxMTFKS0vznDNmzBiNHz9ec+fOlfS3OMvIyNC9996r4cOHa8WKFaqtrfV8KwwAAKBVAXTx4sUrHjSWpN69e1/TR2AHDhzQ6NGjPeuXv16fkZGhvLw8LVy4ULW1tXriiSdUXV2tkSNHqqCgQKGhoZ5zjh07prNnz3rWJ02apKqqKi1ZskR2u11Dhw5VQUFBk/MCAAAzWdxut/taTxozZox69uyptWvXemLkr3/9qzIyMvTtt99qx44dPh+0PTmdTtlsNjkcjqB/HihhwVp/jwAACAIlr03x9wjX7Vr+frfqHaAVK1Zo7NixuuWWWxQfHy9J+uMf/yir1aqPP/64NZcEAABoN60KoCFDhujIkSN69913Pb+vk56ersmTJyssLMynAwIAAPhaqwIoOztbkZGRmjVrltf23NxcVVVVadGiRT4ZDgAAoC206mvwv/3tbxUXF3fF9kGDBmn16tXXPRQAAEBbalUA2e12RUdHX7E9IiJCFRUV1z0UAABAW2pVAMXGxurzzz+/Yvvnn3+umJiY6x4KAACgLbXqGaBZs2Zp3rx5qq+v10MPPSRJKiws1MKFC332S9AAAABtpVUBtGDBAp07d07/9m//prq6OklSaGioFi1apKysLJ8OCAAA4GutCiCLxaJXXnlFixcv1tdff62wsDANGDBAVqvV1/MBAAD4XKsC6LIbb7xRw4YN89UsAAAA7aJVD0EDAAAEMwIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxgn4AOrTp48sFssVy5w5c5o8Pi8v74pjQ0ND23lqAAAQyDr7e4Dv84c//EENDQ2e9UOHDunhhx/WxIkTmz0nPDxchw8f9qxbLJY2nREAAASXgA+giIgIr/WXX35Zt99+u370ox81e47FYlFUVFRbjwYAAIJUwH8E9v+rq6vTunXrNH369Bbf1ampqdFtt92m2NhYPfbYY/rqq69avK7L5ZLT6fRaAABAxxVUAZSfn6/q6mpNnTq12WPuvPNO5ebmasuWLVq3bp0aGxs1YsQInTp1qtlzsrOzZbPZPEtsbGwbTA8AAAKFxe12u/09xNVKSUlRly5d9OGHH171OfX19Ro4cKDS09O1fPnyJo9xuVxyuVyedafTqdjYWDkcDoWHh1/33P6UsGCtv0cAAASBktem+HuE6+Z0OmWz2a7q73fAPwN02YkTJ7Rjxw69//7713TeDTfcoHvuuUdHjx5t9hir1Sqr1Xq9IwIAgCARNB+BrVmzRr1799aPf/zjazqvoaFBX375paKjo9toMgAAEGyCIoAaGxu1Zs0aZWRkqHNn7zetpkyZoqysLM/6iy++qI8//lh/+ctfdPDgQf385z/XiRMnNHPmzPYeGwAABKig+Ahsx44dKi8v1/Tp06/YV15erpCQv3fc+fPnNWvWLNntdt10001KSEjQnj17dNddd7XnyAAAIIAF1UPQ7eVaHqIKdDwEDQC4GqY9BB0UH4EBAAD4EgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDgEEAAAMA4BBAAAjEMAAQAA4xBAAADAOAQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDgEEAAAMA4BBAAAjEMAAQAA4xBAAADAOAQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDgEEAAAME5AB9CyZctksVi8lri4uBbP2bRpk+Li4hQaGqohQ4Zo27Zt7TQtAAAIFgEdQJI0aNAgVVRUeJbdu3c3e+yePXuUnp6uGTNm6IsvvlBaWprS0tJ06NChdpwYAAAEuoAPoM6dOysqKsqz9OrVq9lj//M//1Njx47VggULNHDgQC1fvlw//OEPtXLlynacGAAABLqAD6AjR44oJiZG/fr10+TJk1VeXt7sscXFxUpOTvbalpKSouLi4hZfw+Vyyel0ei0AAKDjCugASkxMVF5engoKCpSTk6OysjI98MADunDhQpPH2+12RUZGem2LjIyU3W5v8XWys7Nls9k8S2xsrM/uAQAABJ6ADqDU1FRNnDhRd999t1JSUrRt2zZVV1frvffe8+nrZGVlyeFweJaTJ0/69PoAACCwdPb3ANeie/fuuuOOO3T06NEm90dFRamystJrW2VlpaKiolq8rtVqldVq9dmcAAAgsAX0O0D/qKamRseOHVN0dHST+5OSklRYWOi1bfv27UpKSmqP8QAAQJAI6ACaP3++ioqKdPz4ce3Zs0fjx49Xp06dlJ6eLkmaMmWKsrKyPMc/9dRTKigo0Ouvv65vvvlGy5Yt04EDBzR37lx/3QIAAAhAAf0R2KlTp5Senq5z584pIiJCI0eO1N69exURESFJKi8vV0jI3xtuxIgRWr9+vZ5//nk999xzGjBggPLz8zV48GB/3QIAAAhAFrfb7fb3EIHG6XTKZrPJ4XAoPDzc3+Ncl4QFa/09AgAgCJS8NsXfI1y3a/n7HdAfgQEAALQFAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgnIAOoOzsbA0bNkzdunVT7969lZaWpsOHD7d4Tl5eniwWi9cSGhraThMDAIBgENABVFRUpDlz5mjv3r3avn276uvr9cgjj6i2trbF88LDw1VRUeFZTpw40U4TAwCAYNDZ3wO0pKCgwGs9Ly9PvXv3VklJiR588MFmz7NYLIqKimrr8QAAQJAK6HeA/pHD4ZAk9ejRo8XjampqdNtttyk2NlaPPfaYvvrqqxaPd7lccjqdXgsAAOi4giaAGhsbNW/ePN1///0aPHhws8fdeeedys3N1ZYtW7Ru3To1NjZqxIgROnXqVLPnZGdny2azeZbY2Ni2uAUAABAgLG632+3vIa7G7Nmz9fvf/167d+/WLbfcctXn1dfXa+DAgUpPT9fy5cubPMblcsnlcnnWnU6nYmNj5XA4FB4eft2z+1PCgrX+HgEAEARKXpvi7xGum9PplM1mu6q/3wH9DNBlc+fO1UcffaRdu3ZdU/xI0g033KB77rlHR48ebfYYq9Uqq9V6vWMCAIAgEdAfgbndbs2dO1ebN2/WJ598or59+17zNRoaGvTll18qOjq6DSYEAADBKKDfAZozZ47Wr1+vLVu2qFu3brLb7ZIkm82msLAwSdKUKVN08803Kzs7W5L04osv6r777lP//v1VXV2t1157TSdOnNDMmTP9dh8AACCwBHQA5eTkSJJGjRrltX3NmjWaOnWqJKm8vFwhIX9/I+v8+fOaNWuW7Ha7brrpJiUkJGjPnj2666672mtsAAAQ4AI6gK7m+exPP/3Ua/3NN9/Um2++2UYTAQCAjiCgnwECAABoCwQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDgEEAAAMA4BBAAAjEMAAQAA4xBAAADAOAQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDgEEAAAMA4BBAAAjEMAAQAA4xBAAADAOAQQAAAwDgEEAACMQwABAADjEEAAAMA4BBAAADAOAQQAAIxDAAEAAOMQQAAAwDhBEUCrVq1Snz59FBoaqsTERO3fv7/F4zdt2qS4uDiFhoZqyJAh2rZtWztNCgAAgkHAB9DGjRuVmZmppUuX6uDBg4qPj1dKSorOnDnT5PF79uxRenq6ZsyYoS+++EJpaWlKS0vToUOH2nlyAAAQqCxut9vt7yFakpiYqGHDhmnlypWSpMbGRsXGxurJJ5/Us88+e8XxkyZNUm1trT766CPPtvvuu09Dhw7V6tWrr+o1nU6nbDabHA6HwsPDfXMjfpKwYK2/RwAABIGS16b4e4Trdi1/vzu300ytUldXp5KSEmVlZXm2hYSEKDk5WcXFxU2eU1xcrMzMTK9tKSkpys/Pb/Z1XC6XXC6XZ93hcEj62//IYNfg+qu/RwAABIGO8Dfv8j1czXs7AR1AZ8+eVUNDgyIjI722R0ZG6ptvvmnyHLvd3uTxdru92dfJzs7WCy+8cMX22NjYVkwNAEDwsf3mF/4ewWcuXLggm83W4jEBHUDtJSsry+tdo8bGRn377bfq2bOnLBaLHycD4GtOp1OxsbE6efJk0H/EDcCb2+3WhQsXFBMT873HBnQA9erVS506dVJlZaXX9srKSkVFRTV5TlRU1DUdL0lWq1VWq9VrW/fu3Vs3NICgEB4eTgABHdD3vfNzWUB/C6xLly5KSEhQYWGhZ1tjY6MKCwuVlJTU5DlJSUlex0vS9u3bmz0eAACYJ6DfAZKkzMxMZWRk6N5779Xw4cO1YsUK1dbWatq0aZKkKVOm6Oabb1Z2drYk6amnntKPfvQjvf766/rxj3+sDRs26MCBA3rrrbf8eRsAACCABHwATZo0SVVVVVqyZInsdruGDh2qgoICz4PO5eXlCgn5+xtZI0aM0Pr16/X888/rueee04ABA5Sfn6/Bgwf76xYABBCr1aqlS5de8bE3ALME/O8AAQAA+FpAPwMEAADQFgggAABgHAIIAAAYhwACAADGIYAAGGXVqlXq06ePQkNDlZiYqP379/t7JAB+QAABMMbGjRuVmZmppUuX6uDBg4qPj1dKSorOnDnj79EAtDO+Bg/AGImJiRo2bJhWrlwp6W+/LB8bG6snn3xSzz77rJ+nA9CeeAcIgBHq6upUUlKi5ORkz7aQkBAlJyeruLjYj5MB8AcCCIARzp49q4aGBs+vyF8WGRkpu93up6kA+AsBBAAAjEMAATBCr1691KlTJ1VWVnptr6ysVFRUlJ+mAuAvBBAAI3Tp0kUJCQkqLCz0bGtsbFRhYaGSkpL8OBkAfwj4fw0eAHwlMzNTGRkZuvfeezV8+HCtWLFCtbW1mjZtmr9HA9DOCCAAxpg0aZKqqqq0ZMkS2e12DR06VAUFBVc8GA2g4+N3gAAAgHF4BggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcQggAABgHAIIgJGmTp2qtLQ0f48BwE8IIAAAYBwCCAAAGIcAAhAUGhsb9eqrr6p///6yWq269dZb9ctf/lKS9OWXX+qhhx5SWFiYevbsqSeeeEI1NTWecxsaGpSZmanu3burZ8+eWrhwof7xn0FsbGxUdna2+vbtq7CwMMXHx+t3v/udZ//58+c1efJkRUREKCwsTAMGDNCaNWva5+YB+BwBBCAoZGVl6eWXX9bixYv15z//WevXr1dkZKRqa2uVkpKim266SX/4wx+0adMm7dixQ3PnzvWc+/rrrysvL0+5ubnavXu3vv32W23evNnr+tnZ2Vq7dq1Wr16tr776Sk8//bR+/vOfq6ioSJI8r/v73/9eX3/9tXJyctSrV692/X8AwHf41+ABBLwLFy4oIiJCK1eu1MyZM732vf3221q0aJFOnjyprl27SpK2bdumcePG6fTp04qMjFRMTIyefvppLViwQJL03XffqW/fvkpISFB+fr5cLpd69OihHTt2KCkpyXPtmTNn6uLFi1q/fr0effRR9erVS7m5ue134wDaTGd/DwAA3+frr7+Wy+XSmDFjmtwXHx/viR9Juv/++9XY2KjDhw8rNDRUFRUVSkxM9Ozv3Lmz7r33Xs/HYEePHtXFixf18MMPe127rq5O99xzjyRp9uzZmjBhgg4ePKhHHnlEaWlpGjFiRFvcLoB2QAABCHhhYWFtev3Lzwtt3bpVN998s9c+q9UqSUpNTdWJEye0bds2bd++XWPGjNGcOXP0q1/9qk1nA9A2eAYIQMAbMGCAwsLCVFhYeMW+gQMH6o9//KNqa2s92z7//HOFhITozjvvlM1mU3R0tPbt2+fZ/91336mkpMSzftddd8lqtaq8vFz9+/f3WmJjYz3HRUREKCMjQ+vWrdOKFSv01ltvtdEdA2hrvAMEIOCFhoZq0aJFWrhwobp06aL7779fVVVV+uqrrzR58mQtXbpUGRkZWrZsmaqqqvTkk0/qX//1XxUZGSlJeuqpp/Tyyy9rwIABiouL0xtvvKHq6mrP9bt166b58+fr6aefVmNjo0aOHCmHw6HPP/9c4eHhysjI0JIlS5SQkKBBgwbJ5XLpo48+0sCBA/30fwTA9SKAAASFxYsXq3PnzlqyZIlOnz6t6Oho/eIXv9APfvAD/d///Z+eeuopDRs2TD/4wQ80YcIEvfHGG55zn3nmGVVUVCgjI0MhISGaPn26xo8fL4fD4Tlm+fLlioiIUHZ2tv7yl7+oe/fu+uEPf6jnnntOktSlSxdlZWXp+PHjCgsL0wMPPKANGza0+/8HAL7Bt8AAAIBxeAYIAAAYhwACAADGIYAAAIBxCCAAAGAcAggAABiHAAIAAMYhgAAAgHEIIAAAYBwCCAAAGIcAAgAAxiGAAACAcf4fJ6/bN8IwdEMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "status = []\n",
    "for job_id in range(len(commands)):\n",
    "    try:\n",
    "        with open(f\"{log_dir}/job_{job_id:05d}.status\", \"r\") as f:\n",
    "            status.append(int(f.read().strip()))\n",
    "    except Exception as e:\n",
    "        status.append(\"waiting\")\n",
    "status = np.array(status)\n",
    "\n",
    "codes, counts = np.unique(status, return_counts=True)\n",
    "df = pd.DataFrame(dict(codes=codes, counts=counts))\n",
    "ax = sns.barplot(data=df, x='codes', y='counts')\n",
    "ax.bar_label(ax.containers[0], fontsize=10);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89442176-59fa-4071-b723-437fcd6699e9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c45b8afe-38b2-4a6b-a075-096781521ecd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd2d676b-004e-41cf-958d-74f6ad6569be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1db0c207-43d2-4c79-8602-191191d75535",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3a53949-a656-497c-b3bc-733b10c333d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2677c3-1c57-462d-b4cd-1452fbb54b48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3b91fab-bb5d-4cf3-88a5-da7bc1386ce7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "187c7fdf-be2a-4ab3-9d8c-5846bb278b1f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b87eb0da-3b7d-41c1-b656-7810bf958163",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "------------------------------------------------------ 0 ------------------------------------------------------\n",
      "\n",
      "10%|▉         | 978/10000 [06:45<1:00:33,  2.48it/s, loss=0.937]\n",
      "\n",
      "------------------------------------------------------ 1 ------------------------------------------------------\n",
      "\n",
      "12%|█▏        | 14769/120000 [07:25<51:22, 34.14it/s, loss=0.875]\n",
      "\n",
      "------------------------------------------------------ 2 ------------------------------------------------------\n",
      "\n",
      "9%|▉         | 1137/12000 [06:56<1:03:49,  2.84it/s, loss=0.939]\n",
      "\n",
      "------------------------------------------------------ 3 ------------------------------------------------------\n",
      "\n",
      "11%|█         | 344/3200 [06:38<53:04,  1.12s/it, loss=0.938]\n",
      "\n",
      "------------------------------------------------------ 4 ------------------------------------------------------\n",
      "\n",
      "11%|█         | 4237/40000 [06:55<56:10, 10.61it/s, loss=0.939]\n",
      "\n",
      "------------------------------------------------------ 5 ------------------------------------------------------\n",
      "\n",
      "9%|▉         | 74/800 [05:52<55:49,  4.61s/it, loss=0.996]\n"
     ]
    }
   ],
   "source": [
    "for job_id in list(range(len(commands))):\n",
    "    print()\n",
    "    print(f'------------------------------------------------------ {job_id} ------------------------------------------------------')\n",
    "    print()\n",
    "    try:\n",
    "        with open(f\"{log_dir}/job_{job_id:05d}.out\", \"r\") as f:\n",
    "            print(f.readlines()[-1].strip())\n",
    "    except Exception as e:\n",
    "        # pass\n",
    "        break\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "1f545d74-19fc-42d7-a744-de052c2619aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/akarshkumar0101/nca-alife-data/logs/2024_08_27_02_37_03'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1812743a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d61cd2fa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nca-alife-jax",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}