data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n- '0'\\n- '1'\\n- '2'\\n- bio\\n- metal\\n- paper\\n- papper\\n- plastic\\n- platstic\\n- rov\\n- timestamp\\n- unknown\\n- wood\\nnc: 13\\n\\nall      216        433      0.627      0.462      0.428      0.201\\n0        216        102      0.527      0.706      0.659      0.362\\n1        216         32      0.402      0.629      0.496      0.187\\n2        216         34      0.443      0.441      0.388      0.173\\nbio      216         24      0.534      0.621      0.586      0.287\\npaper    216          1          1          0     0.0829     0.0249\\nplastic  216         69      0.633        0.6      0.632      0.406\\nrov      216         14      0.817      0.429      0.458      0.225\\ntimestam 216        156      0.291      0.731      0.423      0.109\\nunknown  216          1          1          0      0.124     0.0373\\n\""
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "- '0'\n",
    "- '1'\n",
    "- '2'\n",
    "- bio\n",
    "- metal\n",
    "- paper\n",
    "- papper\n",
    "- plastic\n",
    "- platstic\n",
    "- rov\n",
    "- timestamp\n",
    "- unknown\n",
    "- wood\n",
    "nc: 13\n",
    "\n",
    "all      216        433      0.627      0.462      0.428      0.201\n",
    "0        216        102      0.527      0.706      0.659      0.362\n",
    "1        216         32      0.402      0.629      0.496      0.187\n",
    "2        216         34      0.443      0.441      0.388      0.173\n",
    "bio      216         24      0.534      0.621      0.586      0.287\n",
    "paper    216          1          1          0     0.0829     0.0249\n",
    "plastic  216         69      0.633        0.6      0.632      0.406\n",
    "rov      216         14      0.817      0.429      0.458      0.225\n",
    "timestam 216        156      0.291      0.731      0.423      0.109\n",
    "unknown  216          1          1          0      0.124     0.0373\n",
    "'''"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DELETING EXRTRA COLUMNS\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "double=[0,1,2,3]\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\test\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "        \n",
    "        # input text file\n",
    "    #inputFile = file\n",
    "    # Opening the given file in read-only mode.\n",
    "    #with open(inputFile, 'r') as filedata:\n",
    "    # Read the file lines using readlines()\n",
    "        #inputFilelines = filedata.readlines()\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            \n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0]) in num:\n",
    "                if(i[1]==' '):\n",
    "                    #print(int(i[1]))\n",
    "                    #print(i[0:2:1])\n",
    "                    li.append(i)\n",
    "                \n",
    "                \n",
    "         \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            filedata.close()\n",
    "        \n",
    "\n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "double=[0,1,2,3]\n",
    "\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\train\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "        \n",
    "        # input text file\n",
    "    #inputFile = file\n",
    "    # Opening the given file in read-only mode.\n",
    "    #with open(inputFile, 'r') as filedata:\n",
    "    # Read the file lines using readlines()\n",
    "        #inputFilelines = filedata.readlines()\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            \n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0]) in num:\n",
    "                if(i[1]==' '):\n",
    "                    #print(int(i[1]))\n",
    "                    #print(i[0:2:1])\n",
    "                    li.append(i)\n",
    "                \n",
    "              \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            filedata.close()\n",
    "    \n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Val\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "double=[0,1,2,3]\n",
    "\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\valid\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "        \n",
    "        # input text file\n",
    "    #inputFile = file\n",
    "    # Opening the given file in read-only mode.\n",
    "    #with open(inputFile, 'r') as filedata:\n",
    "    # Read the file lines using readlines()\n",
    "        #inputFilelines = filedata.readlines()\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            \n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0]) in num:\n",
    "                if(i[1]==' '):\n",
    "                #print(i[0:2:1])\n",
    "                    li.append(i)\n",
    "                \n",
    "                \n",
    "              \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            filedata.close()\n",
    "\n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MERGING COLS\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\train\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0])==3:\n",
    "                li.append('1'+i[1:])\n",
    "            elif int(i[0])==7:\n",
    "                li.append('0'+i[1:])\n",
    "            elif int(i[0])==9:\n",
    "                li.append('2'+i[1:])\n",
    "            else:\n",
    "                li.append(i)\n",
    "                \n",
    "                \n",
    "              \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            filedata.close()\n",
    "    \n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\test\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0])==3:\n",
    "                li.append('1'+i[1:])\n",
    "            elif int(i[0])==7:\n",
    "                li.append('0'+i[1:])\n",
    "            elif int(i[0])==9:\n",
    "                li.append('2'+i[1:])\n",
    "            else:\n",
    "                li.append(i)\n",
    "                \n",
    "                \n",
    "              \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            filedata.close()\n",
    "    \n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "num=[0,1,2,3,7,9]\n",
    "my_list = []\n",
    "\n",
    "\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\valid\\labels\\*.txt'): \n",
    "    li=[]\n",
    "    with open(file,'r') as f:\n",
    "        lines = f.readlines()\n",
    "\n",
    "        inputFilelines=lines\n",
    "            #print(inputFilelines)\n",
    "            # storing the current line number in a variable\n",
    "            #lineindex = 1\n",
    "            # Enter the line number to be deleted\n",
    "\n",
    "        for i in inputFilelines:\n",
    "            #print(i)\n",
    "            #count=1\n",
    "            if int(i[0])==3:\n",
    "                li.append('1'+i[1:])\n",
    "            elif int(i[0])==7:\n",
    "                li.append('0'+i[1:])\n",
    "            elif int(i[0])==9:\n",
    "                li.append('2'+i[1:])\n",
    "            else:\n",
    "                li.append(i)\n",
    "                \n",
    "                \n",
    "              \n",
    "        with open(file, 'w') as filedata:\n",
    "            for textline in li:\n",
    "                filedata.write(textline)\n",
    "            \n",
    "            filedata.close()\n",
    "    \n",
    "                \n",
    "        \n",
    "        #for line in lines:\n",
    "            #my_list.append(line.strip()[0])\n",
    "        \n",
    "        \n",
    "#101373 test\n",
    "#111133 train\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_list = []\n",
    "for file in glob.glob(r'C:\\Users\\KARAN\\Desktop\\roboflow dataset\\train\\labels\\*.txt'): \n",
    "\n",
    "    with open(file) as f:\n",
    "        lines = f.readlines()\n",
    "        \n",
    "        for line in lines:\n",
    "            my_list.append(line)\n",
    "#101373 test\n",
    "#111133 train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,len(my_list)):\n",
    "    my_list[i]=int(my_list[i][0:2])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1279\n",
      "609\n",
      "414\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,20):\n",
    "    print(my_list.count(i))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1279\\n609\\n414 \\n'"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "767\n",
    "1615\n",
    "268\n",
    "206\n",
    "0\n",
    "2\n",
    "2\n",
    "512\n",
    "2\n",
    "146\n",
    "'''\n",
    "#ORIGINAL VALUES\n",
    "'''\n",
    "1279\n",
    "609\n",
    "414 \n",
    "'''\n",
    "#After cleaning\n",
    "#TRAIN VALUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a914070154ba479ac7d9039674f9032c209b796f6cf0d596ea89e3ac34911a97"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}