sleep 1 sec if HTTPError to prevent ip blocking

deedy5 · May 17, 2023 · 0215a23 · 0215a23
1 parent baffcf1
commit 0215a23
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py
@@ -48,7 +48,7 @@ def __init__(self, headers=None, proxies=None, timeout=10):
         self._timeout = timeout
 
     def _get_url(self, method, url, **kwargs):
-        for _ in range(2):
+        for _ in range(3):
             try:
                 resp = self._session.request(
                     method, url, timeout=self._timeout, proxies=self._proxies, **kwargs
@@ -58,6 +58,8 @@ def _get_url(self, method, url, **kwargs):
                     sleep(5)
                 resp.raise_for_status()
                 return resp
+            except requests.HTTPError:
+                sleep(1)
             except Exception as ex:
                 logger.debug(f"_get_url() url={url} {type(ex).__name__}")
             sleep(0.25)