KeksPirates/SoftwareManager
1
1
Fork 0
mirror of https://github.com/KeksPirates/SoftwareManager.git synced 2026-08-04 17:59:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

tried to made uztracker scraper async

Browse Source 
not currently working
This commit is contained in:
KeksNino
2025-10-09 14:17:37 +02:00
parent cdb9257859
commit 662751d7db
2 changed files with 29 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data/scrapers/uztracker.py
+23 -20
View File
@@ -1,5 +1,6 @@
import requests import requests
import asyncio import asyncio
import aiohttp
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
global url_uztracker global url_uztracker
@@ -23,7 +24,7 @@ async def init_uztracker():
asyncio.run(init_uztracker()) asyncio.run(init_uztracker())
def scrape_uztracker(search, debug, max_results=450): async def scrape_uztracker(search, debug, max_results=450):
if up: if up:
result = False result = False
global results global results
@@ -38,27 +39,29 @@ def scrape_uztracker(search, debug, max_results=450):
if debug: if debug:
print(search_url) print(search_url)
try: async with aiohttp.ClientSession() as session:
resultCount = 0 try:
response = requests.get(search_url) async with session.get(search_url) as response:
response.raise_for_status() response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser') text = await response.text()
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic')) soup = BeautifulSoup(text, 'html.parser')
for link in links: links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
if link.b: resultCount = 0
resultCount += 1 for link in links:
results.append(link['href']) if link.b:
resulttitles.append(link.b.text) resultCount += 1
result = True results.append(link['href'])
resulttitles.append(link.b.text)
result = True
if resultCount == 0: if resultCount == 0:
# add popup in gui for no result # add popup in gui for no result
break break
except requests.RequestException as e: except aiohttp.ClientError as e:
if result: if result:
print(f"Failed to fetch {search_url}: {e}") print(f"Failed to fetch {search_url}: {e}")
return None return None
else: else:
print("Error: Uztracker down") print("Error: Uztracker down")
core/interface/gui.py
+5 -4
View File
@@ -19,6 +19,7 @@ from PySide6.QtWidgets import (
from PySide6.QtGui import QIcon, QAction from PySide6.QtGui import QIcon, QAction
import darkdetect import darkdetect
import threading import threading
import asyncio
from core.data.scrapers.uztracker import scrape_uztracker from core.data.scrapers.uztracker import scrape_uztracker
from core.data.scrapers.rutracker import scrape_rutracker from core.data.scrapers.rutracker import scrape_rutracker
from core.data.utils import get_magnet_link from core.data.utils import get_magnet_link
@@ -124,9 +125,9 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
if darkdetect.isDark(): if darkdetect.isDark():
settings_action = QAction(QIcon("core/interface/assets/settings_white.png"), "Settings", self)
else:
settings_action = QAction(QIcon("core/interface/assets/settings_dark.png"), "Settings", self) settings_action = QAction(QIcon("core/interface/assets/settings_dark.png"), "Settings", self)
else:
settings_action = QAction(QIcon("core/interface/assets/settings.png"), "Settings", self)
settings_action.triggered.connect(self.settings_dialog) settings_action.triggered.connect(self.settings_dialog)
toolbar.addAction(settings_action) toolbar.addAction(settings_action)
@@ -259,9 +260,9 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
if debug: if debug:
print("User searched for:", search_text) print("User searched for:", search_text)
if tracker == "uztracker": if tracker == "uztracker":
response = scrape_uztracker(search_text, debug) response = asyncio.run(scrape_uztracker(search_text, debug))
if tracker == "rutracker": if tracker == "rutracker":
response = scrape_rutracker(search_text, debug, api_url) response = asyncio.run(scrape_rutracker(search_text, debug, api_url))
if response: if response:
self.postnames, self.postlinks = response self.postnames, self.postlinks = response
self.softwareList.clear() self.softwareList.clear()