Код для парсинга таблицы и данных для динамики (1) (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# plt.style.use('ggplot')    # стиль графиков\n",
    "%matplotlib inline\n",
    "import requests      # Библиотека для отправки запросов\n",
    "import numpy as np   # Библиотека для матриц, векторов и линала\n",
    "import pandas as pd  # Библиотека для табличек\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm.notebook import tqdm\n",
    "from fake_useragent import UserAgent\n",
    "from collections import defaultdict\n",
    "import json\n",
    "parsing_dict = {'Россия': 'russia', 'Великобритания': 'united-kingdom', 'США': 'united-states', 'Германия': 'germany',\n",
    "                    'Франция': 'france', 'Япония': 'japan'}\n",
    "for country in parsing_dict:\n",
    "    if country == \"Япония\":\n",
    "        url = f'https://ru.investing.com/equities/{parsing_dict[country]}'\n",
    "response = requests.get(url, headers={'User-Agent': UserAgent().chrome})\n",
    "tree = response.content\n",
    "df = pd.read_html(tree, encoding='utf-8', thousands = '.')\n",
    "df = df[0]\n",
    "df = df.drop(['Unnamed: 0'], axis=1)\n",
    "df = df.drop(['Unnamed: 9'], axis=1)\n",
    "columns_names = ['Название компании', 'Текущая цена', 'Максимальная цена за сегодня',\n",
    "                     'Минимальная цена за сегодня',\n",
    "                     'Изменение в базисных пунктах', 'Изменение в процентах', 'Объем торгов', 'Время']\n",
    "df.columns = columns_names\n",
    "dates = []\n",
    "df['Текущая цена']= df['Текущая цена'].str.replace('.', '')\n",
    "df['Текущая цена']= df['Текущая цена'].str.replace(',', '.')\n",
    "df['Максимальная цена за сегодня']= df['Максимальная цена за сегодня'].str.replace('.', '')\n",
    "df['Максимальная цена за сегодня']= df['Максимальная цена за сегодня'].str.replace(',', '.')\n",
    "df['Минимальная цена за сегодня']= df['Минимальная цена за сегодня'].str.replace('.', '')\n",
    "df['Минимальная цена за сегодня']= df['Минимальная цена за сегодня'].str.replace(',', '.')\n",
    "lst = []\n",
    "for i in df['Объем торгов']:\n",
    "    i = i.replace(',', '.')\n",
    "    if i[-1] == 'K':\n",
    "        i = float(i[:-1:]) * 1000\n",
    "        lst.append(i)\n",
    "    elif i[-1] == 'M':\n",
    "        i = float(i[:-1:]) * 1000000\n",
    "        lst.append(i)\n",
    "    elif i[-1] == 'B':\n",
    "        lst.append(None)\n",
    "\n",
    "df[\"Объём торгов (для графиков)\"] = lst\n",
    "df['Объём торгов (для графиков)'] = df['Объём торгов (для графиков)']/ 1000\n",
    "df.dropna(axis = 0, inplace=True)\n",
    "df['Текущая цена']= df['Текущая цена'].astype(float)\n",
    "df.to_csv('Shares.csv', 'w', index = False, index_label = False)\n",
    "#Ссылки на каждую акцию\n",
    "links_tree = BeautifulSoup(tree)\n",
    "links_tree = links_tree.find_all('td', {'class': 'bold left noWrap elp plusIconTd'})\n",
    "d = defaultdict()\n",
    "for link in range(1,len(links_tree)+1):\n",
    "    d[link] = 'https://ru.investing.com' + links_tree[link-1].a.get('href')\n",
    "with open('links.json', 'w') as json_file:\n",
    "    json.dump(d, json_file)\n",
    "        \n",
    "#Функция для вывода динамики по акции\n",
    "def dynamics_func(current_share):\n",
    "    with open('links.json') as json_file:\n",
    "        dictionary = json.load(json_file)\n",
    "    #Парсинг таблицы\n",
    "        url = dictionary[f'{current_share}']\n",
    "        response = requests.get(url)\n",
    "        tree = BeautifulSoup(response.content, features=\"lxml\")\n",
    "        data = tree.find('div', {'class': 'instrument-page_section__79xMl'})\n",
    "        data = tree.find('ul', {'class': 'list_list--horizontal__3fUDd navbar_navbar-sub__1_y2w'})\n",
    "        data_list = []\n",
    "        for j in data:\n",
    "            data_list.append(j)\n",
    "        data = 'https://ru.investing.com' + data_list[2].a.get('href')\n",
    "        response = requests.get(data, headers={'User-Agent': UserAgent().chrome})\n",
    "        tree = response.content\n",
    "        df = pd.read_html(tree, attrs={'id': 'curr_table'}, decimal=',', thousands='.')\n",
    "        df = df[0]\n",
    "        # Преобразование даты\n",
    "        dates = []\n",
    "        for ind in range(len(df['Дата'])):\n",
    "            if len(df['Дата'][ind].astype(str)[:-4]) == 4:\n",
    "                date = df['Дата'][ind].astype(str)[:-4][:2]\n",
    "                month = df['Дата'][ind].astype(str)[:-4][-2:]\n",
    "                final_date = date + '.' + month\n",
    "                dates.append(final_date)\n",
    "            elif len(df['Дата'][ind].astype(str)[:-4]) == 3:\n",
    "                date = '0' + df['Дата'][ind].astype(str)[:-4][:1]\n",
    "                month = df['Дата'][ind].astype(str)[:-4][-2:]\n",
    "                final_date = date + '.' + month\n",
    "                dates.append(final_date)\n",
    "        df['Дата'] = dates\n",
    "        return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Дата</th>\n",
       "      <th>Цена</th>\n",
       "      <th>Откр.</th>\n",
       "      <th>Макс.</th>\n",
       "      <th>Мин.</th>\n",
       "      <th>Объём</th>\n",
       "      <th>Изм. %</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16.06</td>\n",
       "      <td>31.88</td>\n",
       "      <td>32.43</td>\n",
       "      <td>32.79</td>\n",
       "      <td>31.74</td>\n",
       "      <td>38,50M</td>\n",
       "      <td>-1,69%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15.06</td>\n",
       "      <td>32.43</td>\n",
       "      <td>32.79</td>\n",
       "      <td>33.29</td>\n",
       "      <td>32.20</td>\n",
       "      <td>53,48M</td>\n",
       "      <td>-0,73%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.06</td>\n",
       "      <td>32.67</td>\n",
       "      <td>32.34</td>\n",
       "      <td>32.83</td>\n",
       "      <td>32.34</td>\n",
       "      <td>11,05M</td>\n",
       "      <td>1,12%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.06</td>\n",
       "      <td>32.30</td>\n",
       "      <td>32.23</td>\n",
       "      <td>32.58</td>\n",
       "      <td>32.12</td>\n",
       "      <td>19,24M</td>\n",
       "      <td>0,07%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.06</td>\n",
       "      <td>32.28</td>\n",
       "      <td>32.40</td>\n",
       "      <td>32.61</td>\n",
       "      <td>32.14</td>\n",
       "      <td>16,69M</td>\n",
       "      <td>-0,38%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>09.06</td>\n",
       "      <td>32.40</td>\n",
       "      <td>32.70</td>\n",
       "      <td>32.88</td>\n",
       "      <td>32.10</td>\n",
       "      <td>36,52M</td>\n",
       "      <td>-1,21%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>08.06</td>\n",
       "      <td>32.80</td>\n",
       "      <td>32.55</td>\n",
       "      <td>32.90</td>\n",
       "      <td>32.45</td>\n",
       "      <td>26,11M</td>\n",
       "      <td>0,99%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>07.06</td>\n",
       "      <td>32.48</td>\n",
       "      <td>31.91</td>\n",
       "      <td>32.63</td>\n",
       "      <td>31.90</td>\n",
       "      <td>26,44M</td>\n",
       "      <td>1,79%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>04.06</td>\n",
       "      <td>31.91</td>\n",
       "      <td>32.30</td>\n",
       "      <td>32.40</td>\n",
       "      <td>31.55</td>\n",
       "      <td>26,28M</td>\n",
       "      <td>-1,05%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>03.06</td>\n",
       "      <td>32.25</td>\n",
       "      <td>31.99</td>\n",
       "      <td>32.55</td>\n",
       "      <td>31.70</td>\n",
       "      <td>62,14M</td>\n",
       "      <td>1,17%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>02.06</td>\n",
       "      <td>31.87</td>\n",
       "      <td>32.20</td>\n",
       "      <td>32.32</td>\n",
       "      <td>31.30</td>\n",
       "      <td>49,96M</td>\n",
       "      <td>-0,98%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>01.06</td>\n",
       "      <td>32.19</td>\n",
       "      <td>31.92</td>\n",
       "      <td>32.31</td>\n",
       "      <td>31.58</td>\n",
       "      <td>36,27M</td>\n",
       "      <td>1,42%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>31.05</td>\n",
       "      <td>31.74</td>\n",
       "      <td>31.85</td>\n",
       "      <td>31.96</td>\n",
       "      <td>31.45</td>\n",
       "      <td>18,90M</td>\n",
       "      <td>-0,25%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>28.05</td>\n",
       "      <td>31.82</td>\n",
       "      <td>31.00</td>\n",
       "      <td>31.85</td>\n",
       "      <td>30.61</td>\n",
       "      <td>55,31M</td>\n",
       "      <td>2,93%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>27.05</td>\n",
       "      <td>30.91</td>\n",
       "      <td>31.78</td>\n",
       "      <td>31.78</td>\n",
       "      <td>30.32</td>\n",
       "      <td>62,84M</td>\n",
       "      <td>-2,45%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>26.05</td>\n",
       "      <td>31.69</td>\n",
       "      <td>31.68</td>\n",
       "      <td>31.86</td>\n",
       "      <td>31.36</td>\n",
       "      <td>28,83M</td>\n",
       "      <td>-0,10%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>25.05</td>\n",
       "      <td>31.72</td>\n",
       "      <td>32.79</td>\n",
       "      <td>32.93</td>\n",
       "      <td>31.42</td>\n",
       "      <td>69,33M</td>\n",
       "      <td>-2,85%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>24.05</td>\n",
       "      <td>32.65</td>\n",
       "      <td>32.65</td>\n",
       "      <td>32.97</td>\n",
       "      <td>32.33</td>\n",
       "      <td>30,36M</td>\n",
       "      <td>0,83%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>21.05</td>\n",
       "      <td>32.38</td>\n",
       "      <td>32.10</td>\n",
       "      <td>32.69</td>\n",
       "      <td>32.01</td>\n",
       "      <td>21,80M</td>\n",
       "      <td>1,05%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20.05</td>\n",
       "      <td>32.05</td>\n",
       "      <td>32.04</td>\n",
       "      <td>32.24</td>\n",
       "      <td>31.61</td>\n",
       "      <td>26,95M</td>\n",
       "      <td>0,00%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>19.05</td>\n",
       "      <td>32.04</td>\n",
       "      <td>33.05</td>\n",
       "      <td>33.20</td>\n",
       "      <td>31.28</td>\n",
       "      <td>81,95M</td>\n",
       "      <td>-3,60%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>18.05</td>\n",
       "      <td>33.24</td>\n",
       "      <td>31.95</td>\n",
       "      <td>33.37</td>\n",
       "      <td>31.90</td>\n",
       "      <td>46,20M</td>\n",
       "      <td>4,23%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>17.05</td>\n",
       "      <td>31.89</td>\n",
       "      <td>32.35</td>\n",
       "      <td>32.45</td>\n",
       "      <td>31.72</td>\n",
       "      <td>28,31M</td>\n",
       "      <td>-1,45%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Дата   Цена  Откр.  Макс.   Мин.   Объём  Изм. %\n",
       "0   16.06  31.88  32.43  32.79  31.74  38,50M  -1,69%\n",
       "1   15.06  32.43  32.79  33.29  32.20  53,48M  -0,73%\n",
       "2   14.06  32.67  32.34  32.83  32.34  11,05M   1,12%\n",
       "3   11.06  32.30  32.23  32.58  32.12  19,24M   0,07%\n",
       "4   10.06  32.28  32.40  32.61  32.14  16,69M  -0,38%\n",
       "5   09.06  32.40  32.70  32.88  32.10  36,52M  -1,21%\n",
       "6   08.06  32.80  32.55  32.90  32.45  26,11M   0,99%\n",
       "7   07.06  32.48  31.91  32.63  31.90  26,44M   1,79%\n",
       "8   04.06  31.91  32.30  32.40  31.55  26,28M  -1,05%\n",
       "9   03.06  32.25  31.99  32.55  31.70  62,14M   1,17%\n",
       "10  02.06  31.87  32.20  32.32  31.30  49,96M  -0,98%\n",
       "11  01.06  32.19  31.92  32.31  31.58  36,27M   1,42%\n",
       "12  31.05  31.74  31.85  31.96  31.45  18,90M  -0,25%\n",
       "13  28.05  31.82  31.00  31.85  30.61  55,31M   2,93%\n",
       "14  27.05  30.91  31.78  31.78  30.32  62,84M  -2,45%\n",
       "15  26.05  31.69  31.68  31.86  31.36  28,83M  -0,10%\n",
       "16  25.05  31.72  32.79  32.93  31.42  69,33M  -2,85%\n",
       "17  24.05  32.65  32.65  32.97  32.33  30,36M   0,83%\n",
       "18  21.05  32.38  32.10  32.69  32.01  21,80M   1,05%\n",
       "19  20.05  32.05  32.04  32.24  31.61  26,95M   0,00%\n",
       "20  19.05  32.04  33.05  33.20  31.28  81,95M  -3,60%\n",
       "21  18.05  33.24  31.95  33.37  31.90  46,20M   4,23%\n",
       "22  17.05  31.89  32.35  32.45  31.72  28,31M  -1,45%"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dynamics_func(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}