refactor: streamline scraper functions to utilize centralized state management

This commit is contained in:
Vxrtrauter
2025-10-09 15:10:18 +02:00
parent 1b0a00068c
commit 1f4fbb845a
5 changed files with 33 additions and 24 deletions
+4 -3
View File
@@ -1,8 +1,9 @@
import requests
from ..utils.state import state
def scrape_rutracker(search_text, debug, server):
search = requests.get(f"{server}/search/{search_text}")
def scrape_rutracker(search_text):
search = requests.get(f"{state.api_url}/search/{search_text}")
if search:
try:
data = search.json()
@@ -10,7 +11,7 @@ def scrape_rutracker(search_text, debug, server):
resultlinks = data["links"]
return resulttitles, resultlinks
except Exception:
if debug:
if state.debug:
print("No results found / No response from server")
return None
+6 -4
View File
@@ -2,6 +2,8 @@ import requests
import asyncio
import aiohttp
from bs4 import BeautifulSoup
from ..utils.state import state
global url_uztracker
url_uztracker = "https://uztracker.net/tracker.php?nm="
@@ -24,7 +26,7 @@ async def init_uztracker():
asyncio.run(init_uztracker())
async def scrape_uztracker(search, debug, max_results=450):
async def scrape_uztracker(search, max_results=450):
if up:
result = False
global results
@@ -36,7 +38,7 @@ async def scrape_uztracker(search, debug, max_results=450):
for start in range(0, max_results, per_page):
search_url = url_uztracker + search + f"&start={start}"
if debug:
if state.debug:
print(search_url)
async with aiohttp.ClientSession() as session:
@@ -72,7 +74,7 @@ async def scrape_uztracker(search, debug, max_results=450):
return resulttitles, results
def get_post_title(post_url, debug):
def get_post_title(post_url):
if up:
response = requests.get(post_url)
soup = BeautifulSoup(response.text, 'html.parser')
@@ -80,7 +82,7 @@ def get_post_title(post_url, debug):
if maintitle:
return maintitle.text
else:
if debug:
if state.debug:
print("Program not found")
else:
print("Error: Uztracker down")
+11
View File
@@ -0,0 +1,11 @@
from dataclasses import dataclass
@dataclass
class AppState:
debug: bool = False
tracker: str = "rutracker"
api_url: str = "https://api.michijackson.xyz"
state = AppState()
+12 -12
View File
@@ -24,17 +24,17 @@ from core.data.scrapers.uztracker import scrape_uztracker
from core.data.scrapers.rutracker import scrape_rutracker
from core.data.utils.tracker import get_item_index
from core.data.utils.settings import save_settings
from core.network.aria2_integration import dlprogress, set_threads
from core.data.utils.state import state
from core.network.aria2_integration import dlprogress
def state_debug(setting):
global debug
if setting is True:
debug = True
state.debug = True
else:
debug = False
state.debug = False
def pass_aria(aria):
@@ -148,7 +148,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def settings_dialog(self):
if debug:
if state.debug:
print("Settings dialog opened")
dialog = QDialog(self)
dialog.setWindowTitle("Settings")
@@ -223,11 +223,11 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def download_selected(self):
item = self.softwareList.currentItem()
if item is not None:
if debug:
if state.debug:
print(f"network {self.softwareList.currentItem().text()}")
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, debug)))
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, state.debug)))
else:
if debug:
if state.debug:
print("No item selected for download.")
# add gui notification for no item selected
@@ -235,15 +235,15 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def return_pressed(self):
search_text = self.searchbar.text()
if search_text == "":
if debug:
if state.debug:
print("Error: Can't search for nothing")
return
if debug:
if state.debug:
print("User searched for:", search_text)
if tracker == "uztracker":
response = asyncio.run(scrape_uztracker(search_text, debug))
response = asyncio.run(scrape_uztracker(search_text))
if tracker == "rutracker":
response = asyncio.run(scrape_rutracker(search_text, debug, api_url))
response = asyncio.run(scrape_rutracker(search_text))
if response:
self.postnames, self.postlinks = response
self.softwareList.clear()
-5
View File
@@ -8,9 +8,4 @@ def add_magnet(uri):
aria2.add_magnet(uri)
def terminate():
return