mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-04 17:59:43 +02:00
refactor: streamline scraper functions to utilize centralized state management
This commit is contained in:
@@ -1,8 +1,9 @@
|
|||||||
import requests
|
import requests
|
||||||
|
from ..utils.state import state
|
||||||
|
|
||||||
|
|
||||||
def scrape_rutracker(search_text, debug, server):
|
def scrape_rutracker(search_text):
|
||||||
search = requests.get(f"{server}/search/{search_text}")
|
search = requests.get(f"{state.api_url}/search/{search_text}")
|
||||||
if search:
|
if search:
|
||||||
try:
|
try:
|
||||||
data = search.json()
|
data = search.json()
|
||||||
@@ -10,7 +11,7 @@ def scrape_rutracker(search_text, debug, server):
|
|||||||
resultlinks = data["links"]
|
resultlinks = data["links"]
|
||||||
return resulttitles, resultlinks
|
return resulttitles, resultlinks
|
||||||
except Exception:
|
except Exception:
|
||||||
if debug:
|
if state.debug:
|
||||||
print("No results found / No response from server")
|
print("No results found / No response from server")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import requests
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
from ..utils.state import state
|
||||||
|
|
||||||
|
|
||||||
global url_uztracker
|
global url_uztracker
|
||||||
url_uztracker = "https://uztracker.net/tracker.php?nm="
|
url_uztracker = "https://uztracker.net/tracker.php?nm="
|
||||||
@@ -24,7 +26,7 @@ async def init_uztracker():
|
|||||||
|
|
||||||
asyncio.run(init_uztracker())
|
asyncio.run(init_uztracker())
|
||||||
|
|
||||||
async def scrape_uztracker(search, debug, max_results=450):
|
async def scrape_uztracker(search, max_results=450):
|
||||||
if up:
|
if up:
|
||||||
result = False
|
result = False
|
||||||
global results
|
global results
|
||||||
@@ -36,7 +38,7 @@ async def scrape_uztracker(search, debug, max_results=450):
|
|||||||
|
|
||||||
for start in range(0, max_results, per_page):
|
for start in range(0, max_results, per_page):
|
||||||
search_url = url_uztracker + search + f"&start={start}"
|
search_url = url_uztracker + search + f"&start={start}"
|
||||||
if debug:
|
if state.debug:
|
||||||
print(search_url)
|
print(search_url)
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -72,7 +74,7 @@ async def scrape_uztracker(search, debug, max_results=450):
|
|||||||
|
|
||||||
return resulttitles, results
|
return resulttitles, results
|
||||||
|
|
||||||
def get_post_title(post_url, debug):
|
def get_post_title(post_url):
|
||||||
if up:
|
if up:
|
||||||
response = requests.get(post_url)
|
response = requests.get(post_url)
|
||||||
soup = BeautifulSoup(response.text, 'html.parser')
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
@@ -80,7 +82,7 @@ def get_post_title(post_url, debug):
|
|||||||
if maintitle:
|
if maintitle:
|
||||||
return maintitle.text
|
return maintitle.text
|
||||||
else:
|
else:
|
||||||
if debug:
|
if state.debug:
|
||||||
print("Program not found")
|
print("Program not found")
|
||||||
else:
|
else:
|
||||||
print("Error: Uztracker down")
|
print("Error: Uztracker down")
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AppState:
|
||||||
|
debug: bool = False
|
||||||
|
tracker: str = "rutracker"
|
||||||
|
api_url: str = "https://api.michijackson.xyz"
|
||||||
|
|
||||||
|
|
||||||
|
state = AppState()
|
||||||
+12
-12
@@ -24,17 +24,17 @@ from core.data.scrapers.uztracker import scrape_uztracker
|
|||||||
from core.data.scrapers.rutracker import scrape_rutracker
|
from core.data.scrapers.rutracker import scrape_rutracker
|
||||||
from core.data.utils.tracker import get_item_index
|
from core.data.utils.tracker import get_item_index
|
||||||
from core.data.utils.settings import save_settings
|
from core.data.utils.settings import save_settings
|
||||||
from core.network.aria2_integration import dlprogress, set_threads
|
from core.data.utils.state import state
|
||||||
|
from core.network.aria2_integration import dlprogress
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def state_debug(setting):
|
def state_debug(setting):
|
||||||
global debug
|
|
||||||
if setting is True:
|
if setting is True:
|
||||||
debug = True
|
state.debug = True
|
||||||
else:
|
else:
|
||||||
debug = False
|
state.debug = False
|
||||||
|
|
||||||
|
|
||||||
def pass_aria(aria):
|
def pass_aria(aria):
|
||||||
@@ -148,7 +148,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
|
|
||||||
def settings_dialog(self):
|
def settings_dialog(self):
|
||||||
|
|
||||||
if debug:
|
if state.debug:
|
||||||
print("Settings dialog opened")
|
print("Settings dialog opened")
|
||||||
dialog = QDialog(self)
|
dialog = QDialog(self)
|
||||||
dialog.setWindowTitle("Settings")
|
dialog.setWindowTitle("Settings")
|
||||||
@@ -223,11 +223,11 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
def download_selected(self):
|
def download_selected(self):
|
||||||
item = self.softwareList.currentItem()
|
item = self.softwareList.currentItem()
|
||||||
if item is not None:
|
if item is not None:
|
||||||
if debug:
|
if state.debug:
|
||||||
print(f"network {self.softwareList.currentItem().text()}")
|
print(f"network {self.softwareList.currentItem().text()}")
|
||||||
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, debug)))
|
self.run_thread(threading.Thread(target=get_item_index, args=(tracker, self.softwareList.currentItem().text(), self.postnames, self.postlinks, state.debug)))
|
||||||
else:
|
else:
|
||||||
if debug:
|
if state.debug:
|
||||||
print("No item selected for download.")
|
print("No item selected for download.")
|
||||||
# add gui notification for no item selected
|
# add gui notification for no item selected
|
||||||
|
|
||||||
@@ -235,15 +235,15 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
def return_pressed(self):
|
def return_pressed(self):
|
||||||
search_text = self.searchbar.text()
|
search_text = self.searchbar.text()
|
||||||
if search_text == "":
|
if search_text == "":
|
||||||
if debug:
|
if state.debug:
|
||||||
print("Error: Can't search for nothing")
|
print("Error: Can't search for nothing")
|
||||||
return
|
return
|
||||||
if debug:
|
if state.debug:
|
||||||
print("User searched for:", search_text)
|
print("User searched for:", search_text)
|
||||||
if tracker == "uztracker":
|
if tracker == "uztracker":
|
||||||
response = asyncio.run(scrape_uztracker(search_text, debug))
|
response = asyncio.run(scrape_uztracker(search_text))
|
||||||
if tracker == "rutracker":
|
if tracker == "rutracker":
|
||||||
response = asyncio.run(scrape_rutracker(search_text, debug, api_url))
|
response = asyncio.run(scrape_rutracker(search_text))
|
||||||
if response:
|
if response:
|
||||||
self.postnames, self.postlinks = response
|
self.postnames, self.postlinks = response
|
||||||
self.softwareList.clear()
|
self.softwareList.clear()
|
||||||
|
|||||||
@@ -8,9 +8,4 @@ def add_magnet(uri):
|
|||||||
aria2.add_magnet(uri)
|
aria2.add_magnet(uri)
|
||||||
|
|
||||||
|
|
||||||
def terminate():
|
|
||||||
return
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user