mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-03 17:39:42 +02:00
Add timeouts, error handling & logging
Add network timeouts and more robust error handling across scrapers/hosts, improve logging concurrency, and refine download handling: - Hosts/sources: add request timeouts (15s) and exception handling to buzzheavier, gofile, steamrip, uztracker and rutracker; use urllib.parse.quote for search queries and better parse error checks. - GoFile: use Session with try/finally, create guest account with timeout, validate API responses and close session. - Context menu: import DirectDownloadHandle and handle stopping direct downloads separately from libtorrent; simplify file deletion with single try/catch; improve logging on exceptions. - GUI/search flow: move UI state changes (clear table, disable searchbar, hide empty results) into MainWindow._start_search and reduce duplicated UI changes in run_search. - Main: remove duplicate is_running assignment. - Libtorrent: snapshot seeded magnets in update loop to avoid concurrent access; minor indentation change in add_seed. - State/logging: add _log_lock to AppState; protect state.log_buffer with lock in consoleLog and flush_log_buffer and fix flush semantics. - Updater: replace busy-wait with QEventLoop/QTimer polling for DirectDownloadHandle progress, stop timer and quit loop on completion/error, and remove unnecessary sleeps. These changes aim to make network operations more reliable, prevent UI freezes, avoid race conditions when logging, and handle direct vs libtorrent downloads more cleanly.
This commit is contained in:
@@ -1,9 +1,13 @@
|
|||||||
|
from utils.logging.logs import consoleLog
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
def scrape_buzzheavier(url):
|
def scrape_buzzheavier(url):
|
||||||
|
try:
|
||||||
response = requests.get(url)
|
response = requests.get(url, timeout=15)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
except requests.RequestException as e:
|
||||||
|
consoleLog(f"Buzzheavier: Failed to fetch page - {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
download_url = url + '/download'
|
download_url = url + '/download'
|
||||||
|
|
||||||
@@ -13,6 +17,10 @@ def scrape_buzzheavier(url):
|
|||||||
'referer': url
|
'referer': url
|
||||||
}
|
}
|
||||||
|
|
||||||
head_response = requests.head(download_url, headers=headers, allow_redirects=False)
|
try:
|
||||||
|
head_response = requests.head(download_url, headers=headers, allow_redirects=False, timeout=15)
|
||||||
hx_redirect = head_response.headers.get('hx-redirect')
|
hx_redirect = head_response.headers.get('hx-redirect')
|
||||||
return hx_redirect
|
return hx_redirect
|
||||||
|
except requests.RequestException as e:
|
||||||
|
consoleLog(f"Buzzheavier: Failed to fetch download URL - {e}")
|
||||||
|
return None
|
||||||
@@ -23,9 +23,9 @@ def scrape_gofile(url):
|
|||||||
|
|
||||||
content_id = match.group(1)
|
content_id = match.group(1)
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
|
try:
|
||||||
# Create a guest account
|
# Create a guest account
|
||||||
r = session.post(f"{_API_BASE}/accounts", headers={"User-Agent": _USER_AGENT})
|
r = session.post(f"{_API_BASE}/accounts", headers={"User-Agent": _USER_AGENT}, timeout=15)
|
||||||
data = r.json()
|
data = r.json()
|
||||||
if data.get("status") != "ok":
|
if data.get("status") != "ok":
|
||||||
consoleLog("GoFile: Failed to create guest account")
|
consoleLog("GoFile: Failed to create guest account")
|
||||||
@@ -44,7 +44,7 @@ def scrape_gofile(url):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Fetch folder contents
|
# Fetch folder contents
|
||||||
r = session.get(f"{_API_BASE}/contents/{content_id}", headers=headers)
|
r = session.get(f"{_API_BASE}/contents/{content_id}", headers=headers, timeout=15)
|
||||||
data = r.json()
|
data = r.json()
|
||||||
if data.get("status") != "ok":
|
if data.get("status") != "ok":
|
||||||
consoleLog(f"GoFile: API error - {data.get('status')}")
|
consoleLog(f"GoFile: API error - {data.get('status')}")
|
||||||
@@ -62,3 +62,5 @@ def scrape_gofile(url):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
return link, headers
|
return link, headers
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
@@ -2,6 +2,7 @@ from utils.network.jsonhandler import split_data, format_data
|
|||||||
from utils.data.tracker import get_magnet_link
|
from utils.data.tracker import get_magnet_link
|
||||||
from utils.logging.logs import consoleLog
|
from utils.logging.logs import consoleLog
|
||||||
from utils.data.state import state
|
from utils.data.state import state
|
||||||
|
from urllib.parse import quote
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
@@ -11,10 +12,18 @@ class RutrackerScraper:
|
|||||||
is_magnet = True
|
is_magnet = True
|
||||||
|
|
||||||
def search(self, query):
|
def search(self, query):
|
||||||
search = requests.get(f"{state.api_url}/search?q={query}", timeout=15)
|
try:
|
||||||
|
search = requests.get(f"{state.api_url}/search?q={quote(query)}", timeout=15)
|
||||||
|
except requests.RequestException as e:
|
||||||
|
consoleLog(f"Failed to reach server: {e}")
|
||||||
|
return []
|
||||||
consoleLog("Sent request to server")
|
consoleLog("Sent request to server")
|
||||||
if search:
|
if search:
|
||||||
|
try:
|
||||||
_, data, _, success, cached = split_data(search.text)
|
_, data, _, success, cached = split_data(search.text)
|
||||||
|
except (KeyError, ValueError) as e:
|
||||||
|
consoleLog(f"Failed to parse server response: {e}")
|
||||||
|
return []
|
||||||
if cached:
|
if cached:
|
||||||
consoleLog("Server response cached")
|
consoleLog("Server response cached")
|
||||||
if success:
|
if success:
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ class SteamripScraper:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
url = f"https://steamrip.com/games-list-page/"
|
url = f"https://steamrip.com/games-list-page/"
|
||||||
response = requests.get(url)
|
response = requests.get(url, timeout=15)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
text = response.text
|
text = response.text
|
||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
@@ -61,7 +61,7 @@ class SteamripScraper:
|
|||||||
|
|
||||||
def scrape_steamrip_game_downloads(self, url):
|
def scrape_steamrip_game_downloads(self, url):
|
||||||
try:
|
try:
|
||||||
response = requests.get(url)
|
response = requests.get(url, timeout=15)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
consoleLog(f"SteamRip: Failed to fetch game page - {e}")
|
consoleLog(f"SteamRip: Failed to fetch game page - {e}")
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ class UztrackerScraper:
|
|||||||
posts = []
|
posts = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(search_url)
|
response = requests.get(search_url, timeout=15)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
soup = BeautifulSoup(response.text, 'html.parser')
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
links = soup.find_all('tr', class_="tCenter hl-tr", id=lambda x: x and x.startswith('tor_'))
|
links = soup.find_all('tr', class_="tCenter hl-tr", id=lambda x: x and x.startswith('tor_'))
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from PySide6.QtCore import Qt, QPoint, Signal, QThread
|
from PySide6.QtCore import Qt, QPoint, Signal, QThread
|
||||||
|
from network.direct_download.handle import DirectDownloadHandle
|
||||||
from utils.logging.logs import consoleLog, remove_download_log
|
from utils.logging.logs import consoleLog, remove_download_log
|
||||||
from utils.network.download import download_selected
|
from utils.network.download import download_selected
|
||||||
from utils.data.tracker import get_magnet_link
|
from utils.data.tracker import get_magnet_link
|
||||||
@@ -121,7 +122,12 @@ class ContextMenu_Downloads:
|
|||||||
|
|
||||||
remove_download_log(magnet_link)
|
remove_download_log(magnet_link)
|
||||||
|
|
||||||
if state.dl_session:
|
if isinstance(magnetdl, DirectDownloadHandle):
|
||||||
|
try:
|
||||||
|
magnetdl.stop()
|
||||||
|
except Exception as e:
|
||||||
|
consoleLog(f"Exception while stopping direct download: {e}")
|
||||||
|
elif state.dl_session:
|
||||||
try:
|
try:
|
||||||
state.dl_session.remove_torrent(magnetdl)
|
state.dl_session.remove_torrent(magnetdl)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -179,29 +185,24 @@ class ContextMenu_Downloads:
|
|||||||
|
|
||||||
remove_download_log(magnet_link)
|
remove_download_log(magnet_link)
|
||||||
|
|
||||||
if state.dl_session:
|
if isinstance(magnetdl, DirectDownloadHandle):
|
||||||
|
try:
|
||||||
|
magnetdl.stop()
|
||||||
|
except Exception as e:
|
||||||
|
consoleLog(f"Exception while stopping direct download: {e}")
|
||||||
|
elif state.dl_session:
|
||||||
try:
|
try:
|
||||||
state.dl_session.remove_torrent(magnetdl)
|
state.dl_session.remove_torrent(magnetdl)
|
||||||
time.sleep(0.5)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
consoleLog(f"Exception while removing download from LibTorrent: {e}")
|
consoleLog(f"Exception while removing download from LibTorrent: {e}")
|
||||||
|
|
||||||
if download_path and os.path.exists(download_path):
|
if download_path and os.path.exists(download_path):
|
||||||
last_error = None
|
|
||||||
for attempt in range(3):
|
|
||||||
try:
|
try:
|
||||||
if os.path.isfile(download_path) or os.path.isdir(download_path):
|
if os.path.isfile(download_path) or os.path.isdir(download_path):
|
||||||
send2trash(download_path)
|
send2trash(download_path)
|
||||||
consoleLog(f"Deleted files for: {torrent_name}", True)
|
consoleLog(f"Deleted files for: {torrent_name}", True)
|
||||||
last_error = None
|
|
||||||
break
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
last_error = e
|
consoleLog(f"Error deleting files: {e}", True)
|
||||||
if attempt < 2:
|
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
if last_error:
|
|
||||||
consoleLog(f"Error deleting files: {last_error}", True)
|
|
||||||
else:
|
else:
|
||||||
consoleLog(f"Removed entry (files not found): {torrent_name}", True)
|
consoleLog(f"Removed entry (files not found): {torrent_name}", True)
|
||||||
|
|
||||||
|
|||||||
@@ -307,6 +307,10 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
table.setItemDelegateForColumn(i, self._tracker_hover_delegate)
|
table.setItemDelegateForColumn(i, self._tracker_hover_delegate)
|
||||||
|
|
||||||
def _start_search(self):
|
def _start_search(self):
|
||||||
|
self.searchbar.setEnabled(False)
|
||||||
|
state.trackertable.setRowCount(0)
|
||||||
|
self.show_empty_results(False)
|
||||||
|
|
||||||
def _search_thread():
|
def _search_thread():
|
||||||
try:
|
try:
|
||||||
run_search(self)
|
run_search(self)
|
||||||
|
|||||||
@@ -6,9 +6,6 @@ for scraper in SCRAPERS:
|
|||||||
state.trackers[scraper.name] = scraper
|
state.trackers[scraper.name] = scraper
|
||||||
|
|
||||||
def run_search(self) -> None:
|
def run_search(self) -> None:
|
||||||
state.trackertable.setRowCount(0)
|
|
||||||
self.show_empty_results(False)
|
|
||||||
|
|
||||||
search_text = self.searchbar.text()
|
search_text = self.searchbar.text()
|
||||||
# Check for empty search
|
# Check for empty search
|
||||||
if search_text == "":
|
if search_text == "":
|
||||||
@@ -16,7 +13,6 @@ def run_search(self) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
state.posts = []
|
state.posts = []
|
||||||
self.searchbar.setEnabled(False)
|
|
||||||
consoleLog(f"User searched for: {search_text}")
|
consoleLog(f"User searched for: {search_text}")
|
||||||
|
|
||||||
# Get current tracker and call its search function
|
# Get current tracker and call its search function
|
||||||
|
|||||||
@@ -45,7 +45,6 @@ class SingleInstance(QObject):
|
|||||||
self.server.listen(SERVER_NAME)
|
self.server.listen(SERVER_NAME)
|
||||||
self.server.newConnection.connect(self.handle_connection)
|
self.server.newConnection.connect(self.handle_connection)
|
||||||
self.is_running = False
|
self.is_running = False
|
||||||
self.is_running = False
|
|
||||||
|
|
||||||
def handle_connection(self):
|
def handle_connection(self):
|
||||||
socket = self.server.nextPendingConnection()
|
socket = self.server.nextPendingConnection()
|
||||||
|
|||||||
@@ -48,13 +48,14 @@ def update_log(shutdown_event):
|
|||||||
try:
|
try:
|
||||||
with state.downloads_lock:
|
with state.downloads_lock:
|
||||||
items = list(state.active_downloads.items())
|
items = list(state.active_downloads.items())
|
||||||
|
seeded = set(state.seeded_magnets)
|
||||||
for magnet_uri, magnetdl in items:
|
for magnet_uri, magnetdl in items:
|
||||||
if isinstance(magnetdl, dict):
|
if isinstance(magnetdl, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
status = magnetdl.status()
|
status = magnetdl.status()
|
||||||
|
|
||||||
if status.state == lt.torrent_status.seeding and magnet_uri not in updated and magnet_uri not in state.seeded_magnets:
|
if status.state == lt.torrent_status.seeding and magnet_uri not in updated and magnet_uri not in seeded:
|
||||||
consoleLog(f"Marking {status.name} as completed")
|
consoleLog(f"Marking {status.name} as completed")
|
||||||
if hasattr(status, 'info_hashes'):
|
if hasattr(status, 'info_hashes'):
|
||||||
info_hash = str(status.info_hashes.v1)
|
info_hash = str(status.info_hashes.v1)
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ class AppState(QObject):
|
|||||||
self.max_connections: int = 200
|
self.max_connections: int = 200
|
||||||
self.max_downloads: int = 10
|
self.max_downloads: int = 10
|
||||||
self.downloads_lock = threading.RLock()
|
self.downloads_lock = threading.RLock()
|
||||||
|
self._log_lock = threading.Lock()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def image_path(self) -> str:
|
def image_path(self) -> str:
|
||||||
|
|||||||
@@ -187,13 +187,15 @@ def _update_download_completed_by_hash_inner(info_hash, completed) -> DownloadLi
|
|||||||
def set_main_window(window):
|
def set_main_window(window):
|
||||||
state.main_window = window
|
state.main_window = window
|
||||||
|
|
||||||
def flush_log_buffer(): # credits to claude
|
def flush_log_buffer():
|
||||||
if state.log_buffer:
|
with state._log_lock:
|
||||||
|
buffer = list(state.log_buffer)
|
||||||
|
state.log_buffer.clear()
|
||||||
|
if buffer:
|
||||||
try:
|
try:
|
||||||
from interface.gui import MainWindow
|
from interface.gui import MainWindow
|
||||||
for log_entry in state.log_buffer:
|
for log_entry in buffer:
|
||||||
MainWindow.add_log(log_entry)
|
MainWindow.add_log(log_entry)
|
||||||
state.log_buffer = []
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
consoleLog(f"Exception while flushing log buffer: {e}")
|
consoleLog(f"Exception while flushing log buffer: {e}")
|
||||||
|
|
||||||
@@ -205,8 +207,10 @@ def consoleLog(text, printAnyways = False):
|
|||||||
try:
|
try:
|
||||||
from interface.gui import MainWindow
|
from interface.gui import MainWindow
|
||||||
if not MainWindow.add_log(formatted_text):
|
if not MainWindow.add_log(formatted_text):
|
||||||
|
with state._log_lock:
|
||||||
state.log_buffer.append(formatted_text)
|
state.log_buffer.append(formatted_text)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
with state._log_lock:
|
||||||
state.log_buffer.append(formatted_text)
|
state.log_buffer.append(formatted_text)
|
||||||
|
|
||||||
if state.debug or printAnyways:
|
if state.debug or printAnyways:
|
||||||
|
|||||||
@@ -2,13 +2,12 @@ from network.direct_download.handle import DirectDownloadHandle
|
|||||||
from utils.general.shutdown import closehelper
|
from utils.general.shutdown import closehelper
|
||||||
from utils.logging.logs import consoleLog
|
from utils.logging.logs import consoleLog
|
||||||
from utils.data.state import state
|
from utils.data.state import state
|
||||||
from PySide6.QtCore import Qt
|
from PySide6.QtCore import Qt, QTimer, QEventLoop
|
||||||
from PySide6 import QtWidgets
|
from PySide6 import QtWidgets
|
||||||
import libtorrent as lt
|
import libtorrent as lt
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import hashlib
|
import hashlib
|
||||||
import time
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
|
|
||||||
@@ -55,14 +54,19 @@ def download_update(assets: list):
|
|||||||
handle = DirectDownloadHandle(url, filename, tempfile.gettempdir())
|
handle = DirectDownloadHandle(url, filename, tempfile.gettempdir())
|
||||||
handle.start()
|
handle.start()
|
||||||
|
|
||||||
while True:
|
loop = QEventLoop()
|
||||||
QtWidgets.QApplication.processEvents()
|
timer = QTimer()
|
||||||
|
timer.setInterval(100)
|
||||||
|
|
||||||
|
def poll_progress():
|
||||||
status = handle.status()
|
status = handle.status()
|
||||||
|
|
||||||
if status.error:
|
if status.error:
|
||||||
state.down_speed_limit = original_limit
|
state.down_speed_limit = original_limit
|
||||||
progress.close()
|
progress.close()
|
||||||
consoleLog(f"Update download failed: {status.error}")
|
consoleLog(f"Update download failed: {status.error}")
|
||||||
|
timer.stop()
|
||||||
|
loop.quit()
|
||||||
return
|
return
|
||||||
|
|
||||||
if status.total_wanted > 0:
|
if status.total_wanted > 0:
|
||||||
@@ -72,9 +76,15 @@ def download_update(assets: list):
|
|||||||
progress.setLabelText(f"Downloading update... ({speed_mb:.1f} MB/s)")
|
progress.setLabelText(f"Downloading update... ({speed_mb:.1f} MB/s)")
|
||||||
|
|
||||||
if status.state == lt.torrent_status.seeding:
|
if status.state == lt.torrent_status.seeding:
|
||||||
break
|
timer.stop()
|
||||||
|
loop.quit()
|
||||||
|
|
||||||
time.sleep(0.1)
|
timer.timeout.connect(poll_progress)
|
||||||
|
timer.start()
|
||||||
|
loop.exec()
|
||||||
|
|
||||||
|
if handle.status().error:
|
||||||
|
return
|
||||||
|
|
||||||
state.down_speed_limit = original_limit
|
state.down_speed_limit = original_limit
|
||||||
|
|
||||||
@@ -94,5 +104,4 @@ def download_update(assets: list):
|
|||||||
|
|
||||||
closehelper()
|
closehelper()
|
||||||
subprocess.Popen([installer_path, "/VERYSILENT", "/SUPPRESSMSGBOXES", "/SP-", "/CLOSEAPPLICATIONS"])
|
subprocess.Popen([installer_path, "/VERYSILENT", "/SUPPRESSMSGBOXES", "/SP-", "/CLOSEAPPLICATIONS"])
|
||||||
time.sleep(1)
|
|
||||||
os._exit(0)
|
os._exit(0)
|
||||||
Reference in New Issue
Block a user