From 004c30c9a6424cb6d299d7b8e9590f8141c6170f Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 10:03:33 +0300 Subject: [PATCH 01/10] translate() - migrate to httpx, remove requests --- duckduckgo_search/duckduckgo_search.py | 7 ++++--- pyproject.toml | 1 - requirements.txt | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py index 74d6794..f0918c3 100644 --- a/duckduckgo_search/duckduckgo_search.py +++ b/duckduckgo_search/duckduckgo_search.py @@ -11,7 +11,6 @@ from urllib.parse import unquote import httpx -import requests from lxml import html logger = logging.getLogger(__name__) @@ -858,11 +857,13 @@ def translate( payload = { "vqd": vqd, "query": "translate", - "from": from_, "to": to, } + if from_: + payload["from"] = from_ - resp = requests.post( + resp = self._get_url( + "POST", "https://duckduckgo.com/translation.js", params=payload, data=keywords.encode(), diff --git a/pyproject.toml b/pyproject.toml index 41f6db4..4acab88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "click>=8.1.3", "lxml>=4.9.2", "httpx[http2]>=0.24.1", - "requests>=2.31.0", ] dynamic = ["version"] diff --git a/requirements.txt b/requirements.txt index 7489310..bf01ff7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ click>=8.1.3 lxml>=4.9.2 httpx[http2]>=0.24.1 -requests>=2.31.0 From 22081930d5859275b6adfa543d88149750ea905c Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 10:30:42 +0300 Subject: [PATCH 02/10] socks5 proxies - add httpx[socks], update README --- README.md | 12 +++--------- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5136a14..1299e23 100755 --- a/README.md +++ b/README.md @@ -139,17 +139,14 @@ ___ ## Using proxy If you send too many requests the site blocks ip for up to one minute and DDGS will raise an exception. -In this case, you need repeat again after a while or to use a proxy ([httpx documentation](https://www.python-httpx.org/advanced)). +In this case, you need repeat again after a while or to use a proxy ([documentation](https://www.python-httpx.org/advanced/#http-proxying)). You can set a timeout if the proxy takes a long time to respond (default timeout=10). *1. The easiest way. Launch the Tor Browser* ```python3 from duckduckgo_search import DDGS -proxies = { - "all://": "socks5h://localhost:9150", -} -ddgs_text_gen = DDGS(proxies=proxies, timeout=20).text("something you need") +ddgs_text_gen = DDGS(proxies="socks5://localhost:9150", timeout=20).text("something you need") for r in ddgs_text_gen: print(r) ``` @@ -157,10 +154,7 @@ for r in ddgs_text_gen: ```python3 from duckduckgo_search import DDGS -proxies = { - "all://": "https://user:password@geo.iproyal.com:32325", -} -ddgs_text_gen = DDGS(proxies=proxies, timeout=20).text("something you need") +ddgs_text_gen = DDGS(proxies="socks5://user:password@geo.iproyal.com:32325", timeout=20).text("something you need") for r in ddgs_text_gen: print(r) ``` diff --git a/pyproject.toml b/pyproject.toml index 4acab88..f6c810c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ dependencies = [ "click>=8.1.3", "lxml>=4.9.2", - "httpx[http2]>=0.24.1", + "httpx[http2,socks]>=0.24.1", ] dynamic = ["version"] diff --git a/requirements.txt b/requirements.txt index bf01ff7..df1011f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ click>=8.1.3 lxml>=4.9.2 -httpx[http2]>=0.24.1 +httpx[http2,socks]>=0.24.1 From 1c19571f4394ba28afafe82cad5b044141a5b65b Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 15:29:54 +0300 Subject: [PATCH 03/10] random useragent choice --- README.md | 149 +++++++++++-------------- duckduckgo_search/duckduckgo_search.py | 26 ++++- duckduckgo_search/version.py | 2 +- 3 files changed, 87 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 1299e23..9ee35ca 100755 --- a/README.md +++ b/README.md @@ -146,17 +146,17 @@ You can set a timeout if the proxy takes a long time to respond (default timeout ```python3 from duckduckgo_search import DDGS -ddgs_text_gen = DDGS(proxies="socks5://localhost:9150", timeout=20).text("something you need") -for r in ddgs_text_gen: - print(r) +with DDGS(proxies="socks5://localhost:9150", timeout=20) as ddgs: + for r in ddgs.text("something you need"): + print(r) ``` *2. Use any proxy server* (*example with [iproyal residential proxies](https://iproyal.com?r=residential_proxies)*) ```python3 from duckduckgo_search import DDGS -ddgs_text_gen = DDGS(proxies="socks5://user:password@geo.iproyal.com:32325", timeout=20).text("something you need") -for r in ddgs_text_gen: - print(r) +with DDGS(proxies="socks5://user:password@geo.iproyal.com:32325", timeout=20) as ddgs: + for r in ddgs.text("something you need"): + print(r) ``` [Go To TOP](#TOP) @@ -190,25 +190,22 @@ def text( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'live free or die' -ddgs_text_gen = ddgs.text(keywords, region='wt-wt', safesearch='Off', timelimit='y') -for r in ddgs_text_gen: - print(r) +with DDGS() as ddgs: + for r in ddgs.text('live free or die', region='wt-wt', safesearch='Off', timelimit='y'): + print(r) # Searching for pdf files -keywords = 'russia filetype:pdf' -ddgs_text_gen = ddgs.text(keywords, region='wt-wt', safesearch='Off', timelimit='y') -for r in ddgs_text_gen: - print(r) +with DDGS() as ddgs: + for r in ddgs.text('russia filetype:pdf', region='wt-wt', safesearch='Off', timelimit='y'): + print(r) # Using lite backend and limit the number of results to 10 from itertools import islice -ddgs_text_gen = DDGS().text("notes from a dead house", backend="lite") -for r in islice(ddgs_text_gen, 10): - print(r) +with DDGS() as ddgs: + ddgs_gen = ddgs.text("notes from a dead house", backend="lite") + for r in islice(ddgs_gen, 10): + print(r) ``` @@ -232,12 +229,9 @@ def answers(keywords: str) -> Generator[dict, None, None]: ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'sun' -ddgs_answers_gen = ddgs.answers(keywords) -for r in ddgs_answers_gen: - print(r) +with DDGS() as ddgs: + for r in ddgs.answers("sun"): + print(r) ``` [Go To TOP](#TOP) @@ -283,21 +277,20 @@ def images( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'butterfly' -ddgs_images_gen = ddgs.images( - keywords, - region="wt-wt", - safesearch="Off", - size=None, - color="Monochrome", - type_image=None, - layout=None, - license_image=None, -) -for r in ddgs_images_gen: - print(r) +with DDGS() as ddgs: + keywords = 'butterfly' + ddgs_images_gen = ddgs.images( + keywords, + region="wt-wt", + safesearch="Off", + size=None, + color="Monochrome", + type_image=None, + layout=None, + license_image=None, + ) + for r in ddgs_images_gen: + print(r) ``` [Go To TOP](#TOP) @@ -334,19 +327,18 @@ def videos( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'tesla' -ddgs_videos_gen = ddgs.videos( - keywords, - region="wt-wt", - safesearch="Off", - timelimit="w", - resolution="high", - duration="medium", -) -for r in ddgs_videos_gen: - print(r) +with DDGS() as ddgs: + keywords = 'tesla' + ddgs_videos_gen = ddgs.videos( + keywords, + region="wt-wt", + safesearch="Off", + timelimit="w", + resolution="high", + duration="medium", + ) + for r in ddgs_videos_gen: + print(r) ``` @@ -378,17 +370,16 @@ def news( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'How soon the sun will die' -ddgs_news_gen = ddgs.news( - keywords, - region="wt-wt", - safesearch="Off", - timelimit="m", -) -for r in ddgs_news_gen: - print(r) +with DDGS() as ddgs: + keywords = 'How soon the sun will die' + ddgs_news_gen = ddgs.news( + keywords, + region="wt-wt", + safesearch="Off", + timelimit="m", + ) + for r in ddgs_news_gen: + print(r) ``` [Go To TOP](#TOP) @@ -434,15 +425,9 @@ def maps( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'school' -ddgs_maps_gen = ddgs.maps( - keywords, - place="Uganda", -) -for r in ddgs_maps_gen: - print(r) +with DDGS() as ddgs: + for r in ddgs.maps("school", place="Uganda"): + print(r) ``` [Go To TOP](#TOP) @@ -471,11 +456,10 @@ def translate( ```python from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'school' -r = ddgs.translate(keywords, to="de") -print(r) +with DDGS() as ddgs: + keywords = 'school' + r = ddgs.translate(keywords, to="de") + print(r) ``` [Go To TOP](#TOP) @@ -502,12 +486,9 @@ def suggestions( ```python3 from duckduckgo_search import DDGS -ddgs = DDGS() - -keywords = 'fly' -ddgs_suggestions_gen = ddgs.suggestions(keywords) -for r in ddgs_suggestions_gen: - print(r) +with DDGS() as ddgs: + for r in ddgs.suggestions("fly) + print(r) ``` [Go To TOP](#TOP) diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py index f0918c3..15e1ffd 100644 --- a/duckduckgo_search/duckduckgo_search.py +++ b/duckduckgo_search/duckduckgo_search.py @@ -6,8 +6,9 @@ from decimal import Decimal from html import unescape from itertools import cycle +from random import choice from time import sleep -from typing import Deque, Dict, Iterator, Optional, Set +from typing import Deque, Dict, Iterator, Optional, Set, Union from urllib.parse import unquote import httpx @@ -15,12 +16,27 @@ logger = logging.getLogger(__name__) +REGEX_500_IN_URL = re.compile(r"[0-9]{3}-[0-9]{2}.js") +REGEX_STRIP_TAGS = re.compile("<.*?>") + +USERAGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13.4; rv:109.0) Gecko/20100101 Firefox/113.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.57", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 OPR/99.0.0.0", +] HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0", + "User-Agent": choice(USERAGENTS), "Referer": "https://duckduckgo.com/", } -REGEX_500_IN_URL = re.compile(r"[0-9]{3}-[0-9]{2}.js") -REGEX_STRIP_TAGS = re.compile("<.*?>") @dataclass @@ -45,7 +61,7 @@ class DDGS: def __init__( self, headers: Optional[Dict[str, str]] = None, - proxies: Optional[Dict] = None, + proxies: Optional[Union[Dict, str]] = None, timeout: int = 10, ) -> None: self._client = httpx.Client( diff --git a/duckduckgo_search/version.py b/duckduckgo_search/version.py index 85197cb..46f67e7 100755 --- a/duckduckgo_search/version.py +++ b/duckduckgo_search/version.py @@ -1 +1 @@ -__version__ = "3.6.0" +__version__ = "3.7.0" From 8158ff559b5f739c538dd563f0104427350876a4 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 15:48:54 +0300 Subject: [PATCH 04/10] cli: random useragent choice --- duckduckgo_search/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/duckduckgo_search/cli.py b/duckduckgo_search/cli.py index ecc6959..0ec2c0c 100644 --- a/duckduckgo_search/cli.py +++ b/duckduckgo_search/cli.py @@ -4,14 +4,14 @@ import os from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime -from shutil import copyfileobj +from random import choice from urllib.parse import unquote import click import httpx # isort: off -from .duckduckgo_search import DDGS +from .duckduckgo_search import DDGS, USERAGENTS from .version import __version__ # isort: on @@ -89,7 +89,7 @@ def sanitize_keywords(keywords): def download_file(url, dir_path, filename): headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0", + "User-Agent": choice(USERAGENTS), } try: with open(os.path.join(dir_path, filename), "wb") as file: From 5d63fb068cb9d3eead7719771328be95f8e17c52 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 15:51:37 +0300 Subject: [PATCH 05/10] text(backend="lite") - normalize title and body --- duckduckgo_search/duckduckgo_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py index 15e1ffd..ba2ca70 100644 --- a/duckduckgo_search/duckduckgo_search.py +++ b/duckduckgo_search/duckduckgo_search.py @@ -350,9 +350,9 @@ def _text_lite( elif i == 3: result_exists = True yield { + "title": self._normalize(title), "href": href, - "title": title, - "body": body, + "body": self._normalize(body), } if result_exists is False: break From b5603642a0550624bb7c1847ded2a3512d2a98c3 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 16:05:44 +0300 Subject: [PATCH 06/10] update tests --- tests/test_duckduckgo_search.py | 123 ++++++++++++++------------------ 1 file changed, 55 insertions(+), 68 deletions(-) diff --git a/tests/test_duckduckgo_search.py b/tests/test_duckduckgo_search.py index 428083f..9859148 100644 --- a/tests/test_duckduckgo_search.py +++ b/tests/test_duckduckgo_search.py @@ -1,78 +1,72 @@ import os import shutil +from itertools import islice from duckduckgo_search import DDGS from duckduckgo_search.cli import download_results, save_csv, save_json def test_text(): - results_gen = DDGS().text("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 25: - break - assert counter >= 25 + with DDGS() as ddgs: + ddgs_gen = ddgs.text("cat") + results = [x for x in islice(ddgs_gen, 25)] + assert len(results) >= 20 + + +def test_text_html(): + with DDGS() as ddgs: + ddgs_gen = ddgs.text("eagle", backend="html") + results = [x for x in islice(ddgs_gen, 25)] + assert len(results) >= 20 + + +def test_text_lite(): + with DDGS() as ddgs: + ddgs_gen = ddgs.text("dog", backend="lite") + results = [x for x in islice(ddgs_gen, 25)] + assert len(results) >= 20 def test_images(): - results_gen = DDGS().images("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 150: - break - assert counter >= 150 + with DDGS() as ddgs: + ddgs_gen = ddgs.images("airplane") + results = [x for x in islice(ddgs_gen, 150)] + assert len(results) >= 140 def test_videos(): - results_gen = DDGS().videos("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 40: - break - assert counter >= 40 + with DDGS() as ddgs: + ddgs_gen = ddgs.videos("sea") + results = [x for x in islice(ddgs_gen, 50)] + assert len(results) >= 40 def test_news(): - results_gen = DDGS().news("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 40: - break - assert counter >= 40 + with DDGS() as ddgs: + ddgs_gen = ddgs.news("tesla") + results = [x for x in islice(ddgs_gen, 40)] + assert len(results) >= 30 def test_maps(): - results_gen = DDGS().maps("school", place="London") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 40: - break - assert counter >= 40 + with DDGS() as ddgs: + ddgs_gen = ddgs.maps("school", place="London") + results = [x for x in islice(ddgs_gen, 40)] + assert len(results) >= 30 def test_answers(): - results_gen = DDGS().answers("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 1: - break - assert counter >= 1 + with DDGS() as ddgs: + ddgs_gen = ddgs.answers("sun") + results = [x for x in islice(ddgs_gen, 5)] + assert len(results) >= 1 def test_suggestions(): - results_gen = DDGS().suggestions("cat") - counter = 0 - for i, x in enumerate(results_gen): - counter += 1 - if i >= 10: - break - assert counter >= 1 + with DDGS() as ddgs: + ddgs_gen = ddgs.suggestions("moon") + results = [x for x in islice(ddgs_gen, 5)] + assert len(results) >= 1 def test_translate(): @@ -86,16 +80,12 @@ def test_translate(): def test_save_csv(): keywords = "butterfly" - results_gen = DDGS().text(keywords) - results = [] - for r in results_gen: - results.append(r) - if len(results) >= 20: - break - assert len(results) >= 20 + with DDGS() as ddgs: + ddgs_gen = ddgs.text(keywords) + results = [x for x in islice(ddgs_gen, 25)] + assert len(results) >= 22 save_csv(f"{keywords}.csv", results) - save_json(f"{keywords}.json", results) # delete files and folders contains keyword in name not_files = True @@ -110,13 +100,10 @@ def test_save_csv(): def test_save_json(): keywords = "chicago" - results_gen = DDGS().text(keywords) - results = [] - for r in results_gen: - results.append(r) - if len(results) >= 20: - break - assert len(results) >= 20 + with DDGS() as ddgs: + ddgs_gen = ddgs.text(keywords) + results = [x for x in islice(ddgs_gen, 25)] + assert len(results) >= 22 save_json(f"{keywords}.json", results) @@ -133,8 +120,8 @@ def test_save_json(): def test_text_download(): keywords = "maradona" - results = [x for i, x in enumerate(DDGS().text(keywords)) if i <= 10] - assert len(results) >= 10 + results = [x for x in islice(DDGS().text(keywords), 10)] + assert len(results) >= 8 download_results(keywords, results) @@ -159,8 +146,8 @@ def test_text_download(): def test_images_download(): keywords = "real madrid" - results = [x for i, x in enumerate(DDGS().images(keywords)) if i <= 10] - assert len(results) >= 10 + results = [x for x in islice(DDGS().images(keywords), 10)] + assert len(results) >= 8 download_results(keywords, results, images=True) From e9345e33af1eb38069157bfe59d76a92007cc62c Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 16:08:02 +0300 Subject: [PATCH 07/10] cli: create file only after sucsessful response --- duckduckgo_search/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/duckduckgo_search/cli.py b/duckduckgo_search/cli.py index 0ec2c0c..6dbf189 100644 --- a/duckduckgo_search/cli.py +++ b/duckduckgo_search/cli.py @@ -92,8 +92,8 @@ def download_file(url, dir_path, filename): "User-Agent": choice(USERAGENTS), } try: - with open(os.path.join(dir_path, filename), "wb") as file: - with httpx.stream("GET", url, headers=headers) as resp: + with httpx.stream("GET", url, headers=headers) as resp: + with open(os.path.join(dir_path, filename), "wb") as file: for chunk in resp.iter_bytes(): file.write(chunk) logger.info(f"File downloaded {url}") From 600e253f7f7555d3656a20edf6e99424f56b82cc Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 16:23:03 +0300 Subject: [PATCH 08/10] cli: correct sanitize_keywords --- duckduckgo_search/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/duckduckgo_search/cli.py b/duckduckgo_search/cli.py index 6dbf189..068ba93 100644 --- a/duckduckgo_search/cli.py +++ b/duckduckgo_search/cli.py @@ -77,12 +77,14 @@ def print_data(data): def sanitize_keywords(keywords): keywords = ( - keywords.replace(" filetype:", "_") + keywords.replace("filetype", "") + .replace(":", "") .replace('"', "'") - .replace("site:", "") + .replace("site", "") .replace(" ", "_") .replace("/", "_") .replace("\\", "_") + .replace(" ", "") ) return keywords From e4c4252a98c2dd610c5773fedaeff4cab37d903d Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 16:29:26 +0300 Subject: [PATCH 09/10] _get_url() - follow_redirects=True --- duckduckgo_search/duckduckgo_search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py index ba2ca70..359623d 100644 --- a/duckduckgo_search/duckduckgo_search.py +++ b/duckduckgo_search/duckduckgo_search.py @@ -82,7 +82,9 @@ def _get_url( ) -> Optional[httpx._models.Response]: for i in range(3): try: - resp = self._client.request(method, url, **kwargs) + resp = self._client.request( + method, url, follow_redirects=True, **kwargs + ) if self._is_500_in_url(str(resp.url)) or resp.status_code == 202: raise httpx._exceptions.HTTPError("") resp.raise_for_status() From 6b0f9e2a8fdc987f60aa9eb23eacb82bfdad57a9 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 30 May 2023 13:43:59 +0000 Subject: [PATCH 10/10] Update duckduckgo_search.py --- duckduckgo_search/duckduckgo_search.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/duckduckgo_search/duckduckgo_search.py b/duckduckgo_search/duckduckgo_search.py index 359623d..3904080 100644 --- a/duckduckgo_search/duckduckgo_search.py +++ b/duckduckgo_search/duckduckgo_search.py @@ -685,9 +685,7 @@ def suggestions( "q": keywords, "kl": region, } - resp = self._get_url( - "GET", "https://duckduckgo.com/ac", params=payload, follow_redirects=True - ) + resp = self._get_url("GET", "https://duckduckgo.com/ac", params=payload) if resp is None: return None try: