improve monkrus scraper for more results

core/data/scrapers/monkrus.py

@@ -1,29 +1,41 @@
 from bs4 import BeautifulSoup
 import requests
 
-
-
 def scrape_monkrus_telegram(query):
-    telegram_channel = "https://t.me/s/real_monkrus/"
+    post = 100
+    max_posts = 250
     posts: list[dict] = []
+    added = set()
+    
+    while post <= max_posts:
+        url = f"https://t.me/s/real_monkrus/{post}"
+        response = requests.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+        bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
+        
+        if not bubbles:
+            break
+        
+        for bubble in bubbles:
+            post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
+            if not post_txt or not post_txt.b:
+                continue
+            
+            title = post_txt.b.text
+            link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
+            
+            if query.lower() in title.lower() and link:
+                post_url = link["href"]
+                if post_url not in added:
+                    added.add(post_url)
+                    posts.append(dict(
+                        author="m0nkrus",
+                        id=len(posts) + 1,
+                        title=title,
+                        url=post_url
+                    ))
+        
+        post += 22
 
-    response = requests.get(telegram_channel)
+    posts.reverse()
-    soup = BeautifulSoup(response.text, "html.parser")
+    return(posts)
-    bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
-
-    for bubble in bubbles:
-        post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text") 
-
-        title = post_txt.b.text
-        url = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
-
-        if query.lower() in title.lower():
-            posts.append(dict(
-                author = "m0nkrus",
-                id = len(posts) + 1,
-                title = title,
-                url = url["href"]
-            ))
-
-    posts.reverse() # reverse to show most recent posts (bubbles) at top of list
-    return(posts)