ddgo

helpers.py

@@ -1,7 +1,9 @@
 import re
+import os
 import time
 import shelve
 import requests
+import html as _html
 from datetime import datetime
 
 from aiogram.methods import AnswerGuestQuery, SendPoll
@@ -582,22 +584,52 @@ def apply_time_tags(text):
 
 
 def web_search(query):
-    r = requests.get(
-        "https://lite.duckduckgo.com/lite/",
-        params={"q": query},
-        headers={"User-Agent": "Mozilla/5.0"},
-        timeout=10
-    )
-    r.raise_for_status()
-    links = re.findall(r"<a[^>]*class=['\"]result-link['\"][^>]*>(.*?)</a>", r.text, re.DOTALL)
-    snippets = re.findall(r"<td[^>]*class=['\"]result-snippet['\"][^>]*>(.*?)</td>", r.text, re.DOTALL)
+    headers = {"User-Agent": "Mozilla/5.0"}
+    links = []
+    snippets = []
+
+    try:
+        r = requests.get("https://lite.duckduckgo.com/lite/", params={"q": query}, headers=headers, timeout=10)
+        r.raise_for_status()
+        links = re.findall(r"<a[^>]*class=['\"]result-link['\"][^>]*>(.*?)</a>", r.text, re.DOTALL)
+        snippets = re.findall(r"<td[^>]*class=['\"]result-snippet['\"][^>]*>(.*?)</td>", r.text, re.DOTALL)
+        if not links:
+            print(f" -- search lite get пусто: {r.status_code} len={len(r.text)}")
+    except Exception as e:
+        print(f" -- search lite get - {e}")
+
+    if not links:
+        try:
+            r = requests.post("https://lite.duckduckgo.com/lite/", data={"q": query}, headers=headers, timeout=10)
+            r.raise_for_status()
+            links = re.findall(r"<a[^>]*class=['\"]result-link['\"][^>]*>(.*?)</a>", r.text, re.DOTALL)
+            snippets = re.findall(r"<td[^>]*class=['\"]result-snippet['\"][^>]*>(.*?)</td>", r.text, re.DOTALL)
+            if not links:
+                print(f" -- search lite post пусто: {r.status_code} len={len(r.text)}")
+        except Exception as e:
+            print(f" -- search lite post - {e}")
+
+    if not links:
+        try:
+            r = requests.post("https://html.duckduckgo.com/html/", data={"q": query}, headers=headers, timeout=10)
+            r.raise_for_status()
+            links = re.findall(r"<a[^>]*class=['\"]result__a['\"][^>]*>(.*?)</a>", r.text, re.DOTALL)
+            snippets = re.findall(r"<a[^>]*class=['\"]result__snippet['\"][^>]*>(.*?)</a>", r.text, re.DOTALL)
+            if not links:
+                print(f" -- search html post пусто: {r.status_code} len={len(r.text)}")
+        except Exception as e:
+            print(f" -- search html post - {e}")
+
     results = []
     for i in range(min(5, len(links))):
-        title = re.sub(r"<[^>]+>", "", links[i]).strip()
-        snippet = re.sub(r"<[^>]+>", "", snippets[i]).strip()[:300] if i < len(snippets) else ""
+        title = _html.unescape(re.sub(r"<[^>]+>", "", links[i])).strip()
+        snippet = _html.unescape(re.sub(r"<[^>]+>", "", snippets[i])).strip()[:300] if i < len(snippets) else ""
         results.append(f"{title}\n{snippet}")
     if not results:
-        return "ничего не нашлось"
+        try:
+            return wiki_search(query)
+        except Exception:
+            return "ничего не нашлось"
     return "\n\n".join(results)
 
 
@@ -612,7 +644,7 @@ def wiki_search(query):
     results = []
     for p in r.json().get("pages", []):
         title = p.get("title", "")
-        excerpt = re.sub(r"<[^>]+>", "", p.get("excerpt", ""))
+        excerpt = _html.unescape(re.sub(r"<[^>]+>", "", p.get("excerpt", "")))
         results.append(f"{title}\n{excerpt[:400]}")
     if not results:
         return "ничего не нашлось в википедии"