Merge branch 'release/1.2.0'

wuyue92tree · Nov 8, 2018 · 850e030 · 850e030
2 parents cd4ed9d + f61190b
commit 850e030
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # Crwy
 
 [![PyPI Version](https://img.shields.io/pypi/v/Crwy.svg)](https://pypi.python.org/pypi/Crwy)
-[![Build Status](https://travis-ci.org/wuyue92tree/crwy.svg?branch=1.1.4)](https://travis-ci.org/wuyue92tree/crwy)
+[![Build Status](https://travis-ci.org/wuyue92tree/crwy.svg?branch=1.2.0)](https://travis-ci.org/wuyue92tree/crwy)
 
 # 简介
 
@@ -16,7 +16,7 @@ Crwy是一个轻量级的爬虫抓取框架，参考Scrapy框架结构开发而
 # 依赖包
 
  * beautifulsoup4>=4.5.1
- * requests==2.12.0
+ * requests>=2.20.0
  * configparser>=3.5.0
  * SQLAlchemy>=1.0.14
  * pyssdb>=0.1.2
@@ -34,27 +34,31 @@ pip install crwy
 ```
 
 or
-前往下载: https://pypi.python.org/pypi/Crwy/1.1.4/
+前往下载: https://pypi.python.org/pypi/Crwy/1.2.0/
 
 # 使用手册
 
 在这里: http://wuyue92tree.antio.top/opensource/crwy.html
 
 # 友情链接
 
-- https://pypi.python.org/pypi/beautifulsoup4/
-- https://pypi.python.org/pypi/requests/
-- https://pypi.python.org/pypi/configparser/
-- https://pypi.python.org/pypi/SQLAlchemy/
-- https://pypi.python.org/pypi/gevent/
-- https://pypi.python.org/pypi/IMAPClient/
+- https://pypi.org/project/Scrapy/
+- https://pypi.org/project/selenium/
+- https://pypi.org/project/beautifulsoup4/
+- https://pypi.org/project/requests/
+- https://pypi.org/project/configparser/
+- https://pypi.org/project/SQLAlchemy/
+- https://pypi.org/project/gevent/
+- https://pypi.org/project/IMAPClient/
 
 # 更新日志
 
 http://wuyue92tree.antio.top/opensource/crwy.html#更新日志
 
 # TODO
 
+- 完善scrapy_plugs
+- 完善selenium_api
 - 兼容python3
 
 
diff --git a/crwy/VERSION b/crwy/VERSION
@@ -1 +1 @@
-1.1.4
+1.2.0
diff --git a/crwy/utils/scrapy_plugs/middlewares.py b/crwy/utils/scrapy_plugs/middlewares.py
@@ -17,7 +17,7 @@
 
 """
 
-import json
+
 import random
 from scrapy.downloadermiddlewares.retry import RetryMiddleware
 from scrapy_redis.connection import get_redis_from_settings

diff --git a/crwy/utils/selenium_api.py b/crwy/utils/selenium_api.py
@@ -25,7 +25,7 @@
     raise CrwyImportException(
         "You should install selenium first! suggestion: pip install "
         "selenium==3.6.0")
-from selenium.common.exceptions import TimeoutException
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
 from selenium.webdriver import DesiredCapabilities, ActionChains
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
@@ -233,7 +233,7 @@ def is_element_visible(self, element):
             the_element = EC.visibility_of_element_located(element)
             assert the_element(driver)
             flag = True
-        except AssertionError:
+        except (AssertionError, NoSuchElementException):
             self.logger.warning('the element is not visible.')
             flag = False
         except Exception as e:

diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 beautifulsoup4>=4.5.1
 configparser>=3.5.0
-requests==2.12.0
+requests>=2.20.0
 gevent>=1.2.1
 retrying>=1.3.3
 imapclient>=2.0.0
diff --git a/setup.py b/setup.py
@@ -26,7 +26,7 @@
     },
     install_requires=[
         'beautifulsoup4>=4.5.1',
-        'requests==2.12.0',
+        'requests>=2.20.0',
         'configparser>=3.5.0',
         'gevent>=1.2.1'
     ],