mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-03 17:39:42 +02:00
refactor: streamline scraper functions to utilize centralized state management
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
import requests
|
||||
from ..utils.state import state
|
||||
|
||||
|
||||
def scrape_rutracker(search_text, debug, server):
|
||||
search = requests.get(f"{server}/search/{search_text}")
|
||||
def scrape_rutracker(search_text):
|
||||
search = requests.get(f"{state.api_url}/search/{search_text}")
|
||||
if search:
|
||||
try:
|
||||
data = search.json()
|
||||
@@ -10,7 +11,7 @@ def scrape_rutracker(search_text, debug, server):
|
||||
resultlinks = data["links"]
|
||||
return resulttitles, resultlinks
|
||||
except Exception:
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("No results found / No response from server")
|
||||
return None
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ import requests
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup
|
||||
from ..utils.state import state
|
||||
|
||||
|
||||
global url_uztracker
|
||||
url_uztracker = "https://uztracker.net/tracker.php?nm="
|
||||
@@ -24,7 +26,7 @@ async def init_uztracker():
|
||||
|
||||
asyncio.run(init_uztracker())
|
||||
|
||||
async def scrape_uztracker(search, debug, max_results=450):
|
||||
async def scrape_uztracker(search, max_results=450):
|
||||
if up:
|
||||
result = False
|
||||
global results
|
||||
@@ -36,7 +38,7 @@ async def scrape_uztracker(search, debug, max_results=450):
|
||||
|
||||
for start in range(0, max_results, per_page):
|
||||
search_url = url_uztracker + search + f"&start={start}"
|
||||
if debug:
|
||||
if state.debug:
|
||||
print(search_url)
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
@@ -72,7 +74,7 @@ async def scrape_uztracker(search, debug, max_results=450):
|
||||
|
||||
return resulttitles, results
|
||||
|
||||
def get_post_title(post_url, debug):
|
||||
def get_post_title(post_url):
|
||||
if up:
|
||||
response = requests.get(post_url)
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
@@ -80,7 +82,7 @@ def get_post_title(post_url, debug):
|
||||
if maintitle:
|
||||
return maintitle.text
|
||||
else:
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("Program not found")
|
||||
else:
|
||||
print("Error: Uztracker down")
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppState:
|
||||
debug: bool = False
|
||||
tracker: str = "rutracker"
|
||||
api_url: str = "https://api.michijackson.xyz"
|
||||
|
||||
|
||||
state = AppState()
|
||||
+12
-12
@@ -24,17 +24,17 @@ from core.data.scrapers.uztracker import scrape_uztracker
|
||||
from core.data.scrapers.rutracker import scrape_rutracker
|
||||
from core.data.utils.tracker import get_item_index
|
||||
from core.data.utils.settings import save_settings
|
||||
from core.network.aria2_integration import dlprogress, set_threads
|
||||
from core.data.utils.state import state
|
||||
from core.network.aria2_integration import dlprogress
|
||||
|
||||
|
||||
|
||||
|
||||
def state_debug(setting):
|
||||
global debug
|
||||
if setting is True:
|
||||
debug = True
|
||||
state.debug = True
|
||||
else:
|
||||
debug = False
|
||||
state.debug = False
|
||||
|
||||
|
||||
def pass_aria(aria):
|
||||
@@ -148,7 +148,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
|
||||
def settings_dialog(self):
|
||||
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("Settings dialog opened")
|
||||
dialog = QDialog(self)
|
||||
dialog.setWindowTitle("Settings")
|
||||
@@ -223,11 +223,11 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
def download_selected(self):
|
||||
item = self.softwareList.currentItem()
|
||||
if item is not None:
|
||||
if debug:
|
||||
if state.debug:
|
||||
print(f"network {self.softwareList.currentItem().text()}")
|
||||
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, debug)))
|
||||
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, state.debug)))
|
||||
else:
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("No item selected for download.")
|
||||
# add gui notification for no item selected
|
||||
|
||||
@@ -235,15 +235,15 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
def return_pressed(self):
|
||||
search_text = self.searchbar.text()
|
||||
if search_text == "":
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("Error: Can't search for nothing")
|
||||
return
|
||||
if debug:
|
||||
if state.debug:
|
||||
print("User searched for:", search_text)
|
||||
if tracker == "uztracker":
|
||||
response = asyncio.run(scrape_uztracker(search_text, debug))
|
||||
response = asyncio.run(scrape_uztracker(search_text))
|
||||
if tracker == "rutracker":
|
||||
response = asyncio.run(scrape_rutracker(search_text, debug, api_url))
|
||||
response = asyncio.run(scrape_rutracker(search_text))
|
||||
if response:
|
||||
self.postnames, self.postlinks = response
|
||||
self.softwareList.clear()
|
||||
|
||||
@@ -8,9 +8,4 @@ def add_magnet(uri):
|
||||
aria2.add_magnet(uri)
|
||||
|
||||
|
||||
def terminate():
|
||||
return
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user