KeksPirates/SoftwareManager
1
1
Fork 0
mirror of https://github.com/KeksPirates/SoftwareManager.git synced 2026-08-04 09:59:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

improve monkrus scraper for more results

Browse Source
This commit is contained in:
Vxrtrauter
2025-10-18 02:25:15 +02:00
parent c99ce339e0
commit bd8320a26c
1 changed files with 35 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data/scrapers/monkrus.py
+30 -18
View File
@@ -1,29 +1,41 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import requests import requests
def scrape_monkrus_telegram(query): def scrape_monkrus_telegram(query):
telegram_channel = "https://t.me/s/real_monkrus/" post = 100
max_posts = 250
posts: list[dict] = [] posts: list[dict] = []
added = set()
response = requests.get(telegram_channel) while post <= max_posts:
soup = BeautifulSoup(response.text, "html.parser") url = f"https://t.me/s/real_monkrus/{post}"
bubbles = soup.find_all("div", class_="tgme_widget_message_bubble") response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
for bubble in bubbles: if not bubbles:
post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text") break
title = post_txt.b.text for bubble in bubbles:
url = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net")) post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
if not post_txt or not post_txt.b:
continue
if query.lower() in title.lower(): title = post_txt.b.text
posts.append(dict( link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
author = "m0nkrus",
id = len(posts) + 1,
title = title,
url = url["href"]
))
posts.reverse() # reverse to show most recent posts (bubbles) at top of list if query.lower() in title.lower() and link:
post_url = link["href"]
if post_url not in added:
added.add(post_url)
posts.append(dict(
author="m0nkrus",
id=len(posts) + 1,
title=title,
url=post_url
))
post += 22
posts.reverse()
return(posts) return(posts)