From 1822a150d7beafb3a13dc0455f50797d671285a6 Mon Sep 17 00:00:00 2001 From: Vxrtrauter Date: Sat, 18 Oct 2025 01:33:12 +0200 Subject: [PATCH] finnish monkrus scraper (probably) --- core/data/scrapers/monkrus.py | 22 +++++++--------------- core/utils/data/state.py | 7 ------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/core/data/scrapers/monkrus.py b/core/data/scrapers/monkrus.py index d3cdff4..aad7aef 100644 --- a/core/data/scrapers/monkrus.py +++ b/core/data/scrapers/monkrus.py @@ -1,4 +1,4 @@ -from core.utils.data.state import state, Post +from core.utils.data.state import state from bs4 import BeautifulSoup import requests @@ -6,11 +6,9 @@ import requests def scrape_monkrus_telegram(): telegram_channel = "https://t.me/s/real_monkrus/" - posts: list[Post] = [] + posts: list[dict] = [] response = requests.get(telegram_channel) - print(response.status_code) # rm later - soup = BeautifulSoup(response.text, "html.parser") bubbles = soup.find_all("div", class_="tgme_widget_message_bubble") @@ -20,18 +18,12 @@ def scrape_monkrus_telegram(): title = post_txt.b.text url = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net")) - - - posts.append(Post( + posts.append(dict( + author = "m0nkrus", id = len(posts) + 1, title = title, - url = url, - author = "m0nkrus" - + url = url["href"] )) - print(posts) - - - -scrape_monkrus_telegram() \ No newline at end of file + posts.reverse() # reverse to show most recent posts (bubbles) at top of list + return(posts) \ No newline at end of file diff --git a/core/utils/data/state.py b/core/utils/data/state.py index deb9558..633415e 100644 --- a/core/utils/data/state.py +++ b/core/utils/data/state.py @@ -20,10 +20,3 @@ class AppState: aria2_threads: int = 4 state = AppState(posts=[], post_titles=[], post_urls=[], post_author=[], downloads=[], download_path="") - -@dataclass -class Post: - id: int - title: str - url: str - author: str