mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-03 17:39:42 +02:00
add & impl. m0nkrus scraperM
This commit is contained in:
@@ -0,0 +1,41 @@
|
|||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import requests
|
||||||
|
|
||||||
|
def scrape_monkrus_telegram(query):
|
||||||
|
post = 100
|
||||||
|
max_posts = 250
|
||||||
|
posts: list[dict] = []
|
||||||
|
added = set()
|
||||||
|
|
||||||
|
while post <= max_posts:
|
||||||
|
url = f"https://t.me/s/real_monkrus/{post}"
|
||||||
|
response = requests.get(url)
|
||||||
|
soup = BeautifulSoup(response.text, "html.parser")
|
||||||
|
bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
|
||||||
|
|
||||||
|
if not bubbles:
|
||||||
|
break
|
||||||
|
|
||||||
|
for bubble in bubbles:
|
||||||
|
post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
|
||||||
|
if not post_txt or not post_txt.b:
|
||||||
|
continue
|
||||||
|
|
||||||
|
title = post_txt.b.text
|
||||||
|
link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
|
||||||
|
|
||||||
|
if query.lower() in title.lower() and link:
|
||||||
|
post_url = link["href"]
|
||||||
|
if post_url not in added:
|
||||||
|
added.add(post_url)
|
||||||
|
posts.append(dict(
|
||||||
|
author="m0nkrus",
|
||||||
|
id=len(posts) + 1,
|
||||||
|
title=title,
|
||||||
|
url=post_url
|
||||||
|
))
|
||||||
|
|
||||||
|
post += 22
|
||||||
|
|
||||||
|
posts.reverse()
|
||||||
|
return(posts)
|
||||||
@@ -139,7 +139,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
toolbar.setLayoutDirection(Qt.RightToLeft)
|
toolbar.setLayoutDirection(Qt.RightToLeft)
|
||||||
|
|
||||||
self.tracker_list = QComboBox()
|
self.tracker_list = QComboBox()
|
||||||
self.tracker_list.addItems(["rutracker", "uztracker"])
|
self.tracker_list.addItems(["rutracker", "uztracker", "m0nkrus"])
|
||||||
self.tracker_list.activated.connect(self.set_tracker)
|
self.tracker_list.activated.connect(self.set_tracker)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import asyncio
|
|
||||||
from core.data.scrapers.uztracker import scrape_uztracker
|
from core.data.scrapers.uztracker import scrape_uztracker
|
||||||
from core.data.scrapers.rutracker import scrape_rutracker
|
from core.data.scrapers.rutracker import scrape_rutracker
|
||||||
|
from core.data.scrapers.monkrus import scrape_monkrus_telegram
|
||||||
from core.utils.network.jsonhandler import split_data, format_data
|
from core.utils.network.jsonhandler import split_data, format_data
|
||||||
from core.utils.data.state import state
|
from core.utils.data.state import state
|
||||||
|
|
||||||
@@ -55,4 +55,20 @@ def return_pressed(self):
|
|||||||
if state.debug:
|
if state.debug:
|
||||||
print(f"No response from rutracker")
|
print(f"No response from rutracker")
|
||||||
self.softwareList.clear()
|
self.softwareList.clear()
|
||||||
self.show_empty_results(True)
|
self.show_empty_results(True)
|
||||||
|
|
||||||
|
elif state.tracker == "m0nkrus":
|
||||||
|
state.posts = scrape_monkrus_telegram(search_text)
|
||||||
|
|
||||||
|
if state.posts == []:
|
||||||
|
if state.debug:
|
||||||
|
print(f"No Results for {search_text}")
|
||||||
|
self.softwareList.clear()
|
||||||
|
self.show_empty_results(True)
|
||||||
|
else:
|
||||||
|
state.post_titles, _, state.post_author = format_data(state.posts)
|
||||||
|
self.show_empty_results(False)
|
||||||
|
self.post_author_list.clear()
|
||||||
|
self.post_author_list.addItems(state.post_author)
|
||||||
|
self.softwareList.clear()
|
||||||
|
self.softwareList.addItems(state.post_titles)
|
||||||
|
|||||||
@@ -19,6 +19,12 @@ def get_item_url(item, posts, post_titles): # softwarelist currentitem, post lis
|
|||||||
if state.debug:
|
if state.debug:
|
||||||
print("Found post URL: ", post_links[0])
|
print("Found post URL: ", post_links[0])
|
||||||
return post_links[0]
|
return post_links[0]
|
||||||
|
if state.tracker == "m0nkrus":
|
||||||
|
item_dict = posts[post_index]
|
||||||
|
_, post_links, _ = format_data([item_dict])
|
||||||
|
if state.debug:
|
||||||
|
print("Found post URL: ", post_links[0])
|
||||||
|
return post_links[0]
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user