refactor: streamline scraper functions to utilize centralized state management

This commit is contained in:
Vxrtrauter
2025-10-09 15:10:18 +02:00
parent 1b0a00068c
commit 1f4fbb845a
5 changed files with 33 additions and 24 deletions
+4 -3
View File
@@ -1,8 +1,9 @@
import requests import requests
from ..utils.state import state
def scrape_rutracker(search_text, debug, server): def scrape_rutracker(search_text):
search = requests.get(f"{server}/search/{search_text}") search = requests.get(f"{state.api_url}/search/{search_text}")
if search: if search:
try: try:
data = search.json() data = search.json()
@@ -10,7 +11,7 @@ def scrape_rutracker(search_text, debug, server):
resultlinks = data["links"] resultlinks = data["links"]
return resulttitles, resultlinks return resulttitles, resultlinks
except Exception: except Exception:
if debug: if state.debug:
print("No results found / No response from server") print("No results found / No response from server")
return None return None
+6 -4
View File
@@ -2,6 +2,8 @@ import requests
import asyncio import asyncio
import aiohttp import aiohttp
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from ..utils.state import state
global url_uztracker global url_uztracker
url_uztracker = "https://uztracker.net/tracker.php?nm=" url_uztracker = "https://uztracker.net/tracker.php?nm="
@@ -24,7 +26,7 @@ async def init_uztracker():
asyncio.run(init_uztracker()) asyncio.run(init_uztracker())
async def scrape_uztracker(search, debug, max_results=450): async def scrape_uztracker(search, max_results=450):
if up: if up:
result = False result = False
global results global results
@@ -36,7 +38,7 @@ async def scrape_uztracker(search, debug, max_results=450):
for start in range(0, max_results, per_page): for start in range(0, max_results, per_page):
search_url = url_uztracker + search + f"&start={start}" search_url = url_uztracker + search + f"&start={start}"
if debug: if state.debug:
print(search_url) print(search_url)
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -72,7 +74,7 @@ async def scrape_uztracker(search, debug, max_results=450):
return resulttitles, results return resulttitles, results
def get_post_title(post_url, debug): def get_post_title(post_url):
if up: if up:
response = requests.get(post_url) response = requests.get(post_url)
soup = BeautifulSoup(response.text, 'html.parser') soup = BeautifulSoup(response.text, 'html.parser')
@@ -80,7 +82,7 @@ def get_post_title(post_url, debug):
if maintitle: if maintitle:
return maintitle.text return maintitle.text
else: else:
if debug: if state.debug:
print("Program not found") print("Program not found")
else: else:
print("Error: Uztracker down") print("Error: Uztracker down")
+11
View File
@@ -0,0 +1,11 @@
from dataclasses import dataclass
@dataclass
class AppState:
debug: bool = False
tracker: str = "rutracker"
api_url: str = "https://api.michijackson.xyz"
state = AppState()
+12 -12
View File
@@ -24,17 +24,17 @@ from core.data.scrapers.uztracker import scrape_uztracker
from core.data.scrapers.rutracker import scrape_rutracker from core.data.scrapers.rutracker import scrape_rutracker
from core.data.utils.tracker import get_item_index from core.data.utils.tracker import get_item_index
from core.data.utils.settings import save_settings from core.data.utils.settings import save_settings
from core.network.aria2_integration import dlprogress, set_threads from core.data.utils.state import state
from core.network.aria2_integration import dlprogress
def state_debug(setting): def state_debug(setting):
global debug
if setting is True: if setting is True:
debug = True state.debug = True
else: else:
debug = False state.debug = False
def pass_aria(aria): def pass_aria(aria):
@@ -148,7 +148,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def settings_dialog(self): def settings_dialog(self):
if debug: if state.debug:
print("Settings dialog opened") print("Settings dialog opened")
dialog = QDialog(self) dialog = QDialog(self)
dialog.setWindowTitle("Settings") dialog.setWindowTitle("Settings")
@@ -223,11 +223,11 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def download_selected(self): def download_selected(self):
item = self.softwareList.currentItem() item = self.softwareList.currentItem()
if item is not None: if item is not None:
if debug: if state.debug:
print(f"network {self.softwareList.currentItem().text()}") print(f"network {self.softwareList.currentItem().text()}")
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, debug))) self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, state.debug)))
else: else:
if debug: if state.debug:
print("No item selected for download.") print("No item selected for download.")
# add gui notification for no item selected # add gui notification for no item selected
@@ -235,15 +235,15 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
def return_pressed(self): def return_pressed(self):
search_text = self.searchbar.text() search_text = self.searchbar.text()
if search_text == "": if search_text == "":
if debug: if state.debug:
print("Error: Can't search for nothing") print("Error: Can't search for nothing")
return return
if debug: if state.debug:
print("User searched for:", search_text) print("User searched for:", search_text)
if tracker == "uztracker": if tracker == "uztracker":
response = asyncio.run(scrape_uztracker(search_text, debug)) response = asyncio.run(scrape_uztracker(search_text))
if tracker == "rutracker": if tracker == "rutracker":
response = asyncio.run(scrape_rutracker(search_text, debug, api_url)) response = asyncio.run(scrape_rutracker(search_text))
if response: if response:
self.postnames, self.postlinks = response self.postnames, self.postlinks = response
self.softwareList.clear() self.softwareList.clear()
-5
View File
@@ -8,9 +8,4 @@ def add_magnet(uri):
aria2.add_magnet(uri) aria2.add_magnet(uri)
def terminate():
return