mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-03 17:39:42 +02:00
add: direct http download, fix libtorrent speed limit settings
This commit is contained in:
@@ -1,5 +1,4 @@
|
||||
from core.utils.data.state import state
|
||||
from core.utils.network.jsonhandler import format_data
|
||||
from core.utils.data.tracker import get_magnet_link
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import Dict
|
||||
|
||||
+76
-33
@@ -333,7 +333,10 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
self.headers = ["Action", "Name", "Status", "Progress", "Speed", "Size", "Total Size"]
|
||||
|
||||
def rowCount(self, parent=QModelIndex()):
|
||||
return len(state.active_downloads)
|
||||
if parent.isValid():
|
||||
return 0
|
||||
with state.downloads_lock:
|
||||
return len(state.active_downloads)
|
||||
|
||||
def columnCount(self, parent=QModelIndex()):
|
||||
return len(self.headers)
|
||||
@@ -344,20 +347,29 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
return None
|
||||
|
||||
def data(self, index, role=Qt.ItemDataRole.DisplayRole):
|
||||
if role == Qt.ItemDataRole.DisplayRole:
|
||||
col = index.column()
|
||||
if not index.isValid():
|
||||
return None
|
||||
|
||||
with state.downloads_lock:
|
||||
if index.row() >= len(state.active_downloads) or index.row() < 0:
|
||||
return None
|
||||
magnet_link = list(state.active_downloads.keys())[index.row()]
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
status = magnetdl.status()
|
||||
|
||||
try:
|
||||
magnet_link = list(state.active_downloads.keys())[index.row()]
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
status = magnetdl.status()
|
||||
except (IndexError, KeyError, RuntimeError):
|
||||
return None
|
||||
|
||||
if role == Qt.ItemDataRole.DisplayRole:
|
||||
col = index.column()
|
||||
if col == 0:
|
||||
pass
|
||||
elif col == 1:
|
||||
return status.name if status.has_metadata else "Fetching metadata..."
|
||||
elif col == 2:
|
||||
if status.paused:
|
||||
is_auto = status.auto_managed
|
||||
is_auto = getattr(status, 'auto_managed', True)
|
||||
return "Queued" if is_auto else "Paused"
|
||||
elif status.state == lt.torrent_status.downloading:
|
||||
return "Downloading"
|
||||
@@ -402,34 +414,43 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
else:
|
||||
return "∞" if status.paused else "Stalled"
|
||||
if role == Qt.ItemDataRole.UserRole and index.column() == 0:
|
||||
magnet_link = list(state.active_downloads.keys())[index.row()]
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
return magnetdl.status().paused
|
||||
return status.paused
|
||||
return None
|
||||
|
||||
def toggle_pause_resume(self, row):
|
||||
if row >= len(state.active_downloads) or row < 0:
|
||||
return
|
||||
magnet_link = list(state.active_downloads.keys())[row]
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
status = magnetdl.status()
|
||||
with state.downloads_lock:
|
||||
if row >= len(state.active_downloads) or row < 0:
|
||||
return
|
||||
try:
|
||||
magnet_link = list(state.active_downloads.keys())[row]
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
status = magnetdl.status()
|
||||
except (IndexError, KeyError, RuntimeError):
|
||||
return
|
||||
|
||||
if status.state == lt.torrent_status.seeding:
|
||||
save_path = magnetdl.save_path()
|
||||
if save_path and os.path.exists(save_path):
|
||||
if platform.system() == "Windows":
|
||||
os.startfile(os.path.normpath(save_path))
|
||||
elif platform.system() == "Linux":
|
||||
subprocess.Popen(["xdg-open", save_path])
|
||||
elif platform.system() == "Darwin":
|
||||
subprocess.Popen(["open", save_path])
|
||||
try:
|
||||
save_path = magnetdl.save_path()
|
||||
if save_path and os.path.exists(save_path):
|
||||
if platform.system() == "Windows":
|
||||
os.startfile(os.path.normpath(save_path))
|
||||
elif platform.system() == "Linux":
|
||||
subprocess.Popen(["xdg-open", save_path])
|
||||
elif platform.system() == "Darwin":
|
||||
subprocess.Popen(["open", save_path])
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
is_paused = status.paused
|
||||
if is_paused:
|
||||
magnetdl.set_flags(lt.torrent_flags.auto_managed)
|
||||
if hasattr(magnetdl, 'set_flags'):
|
||||
magnetdl.set_flags(lt.torrent_flags.auto_managed)
|
||||
magnetdl.resume()
|
||||
consoleLog(f"Resumed download: {status.name}", True)
|
||||
else:
|
||||
magnetdl.unset_flags(lt.torrent_flags.auto_managed)
|
||||
if hasattr(magnetdl, 'unset_flags'):
|
||||
magnetdl.unset_flags(lt.torrent_flags.auto_managed)
|
||||
magnetdl.pause()
|
||||
consoleLog(f"Paused download: {status.name}", True)
|
||||
idx = self.index(row, 0)
|
||||
@@ -826,7 +847,10 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
def _update_speed_label(self):
|
||||
total_down = 0
|
||||
total_up = 0
|
||||
for handle in state.active_downloads.values():
|
||||
with state.downloads_lock:
|
||||
active_items = list(state.active_downloads.values())
|
||||
|
||||
for handle in active_items:
|
||||
try:
|
||||
s = handle.status()
|
||||
total_down += s.download_rate
|
||||
@@ -837,7 +861,10 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
up_kb = total_up / 1024
|
||||
down_text = f"{down_kb / 1024:.1f} MB/s" if down_kb > 1024 else f"{down_kb:.1f} kB/s"
|
||||
up_text = f"{up_kb / 1024:.1f} MB/s" if up_kb > 1024 else f"{up_kb:.1f} kB/s"
|
||||
self.speed_label.setText(f"↓ {down_text} ↑ {up_text}")
|
||||
|
||||
|
||||
if hasattr(self, 'speed_label'):
|
||||
self.speed_label.setText(f"↓ {down_text} ↑ {up_text}")
|
||||
|
||||
def mousePressEvent(self, event):
|
||||
state.trackertable.clearSelection()
|
||||
@@ -855,6 +882,10 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
|
||||
def eventFilter(self, obj, event):
|
||||
try:
|
||||
if hasattr(self, 'speed_label') and obj == self.speed_label:
|
||||
if event.type() == QEvent.Type.MouseButtonRelease:
|
||||
settings_dialog(self)
|
||||
return True
|
||||
if obj == state.trackertable.viewport():
|
||||
if event.type() == QEvent.Type.MouseMove:
|
||||
pos = event.position().toPoint() if hasattr(event, 'position') else event.pos()
|
||||
@@ -882,7 +913,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
old_row = self._hovered_row
|
||||
self._hovered_row = -1
|
||||
self._invalidate_hover_row(old_row)
|
||||
except RuntimeError:
|
||||
except (RuntimeError, AttributeError):
|
||||
pass
|
||||
return super().eventFilter(obj, event)
|
||||
|
||||
@@ -903,7 +934,9 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
self.emptyResults.hide()
|
||||
|
||||
def show_empty_downloads(self):
|
||||
if len(state.active_downloads) > 0:
|
||||
with state.downloads_lock:
|
||||
has_downloads = len(state.active_downloads) > 0
|
||||
if has_downloads:
|
||||
self.emptyDownload.hide()
|
||||
self.downloadList.show()
|
||||
else:
|
||||
@@ -958,8 +991,13 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
magnetdl = state.active_downloads[magnet_link]
|
||||
confirm = QMessageBox.question(self, "Cancel Download", f"Are you sure you want to cancel the download of '{magnetdl.status().name}'?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
|
||||
if confirm == QMessageBox.StandardButton.Yes:
|
||||
if state.dl_session:
|
||||
state.dl_session.remove_torrent(magnetdl)
|
||||
if hasattr(magnetdl, 'stop'):
|
||||
magnetdl.stop()
|
||||
elif state.dl_session:
|
||||
try:
|
||||
state.dl_session.remove_torrent(magnetdl)
|
||||
except Exception:
|
||||
pass
|
||||
del state.active_downloads[magnet_link]
|
||||
remove_download_log(magnet_link)
|
||||
consoleLog(f"Cancelled download: {magnetdl.status().name}", True)
|
||||
@@ -978,8 +1016,13 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
||||
download_path = os.path.join(save_path, torrent_name)
|
||||
confirm = QMessageBox.question(self, "Delete Files", f"Are you sure you want to delete the downloaded files of '{magnetdl.status().name}'? This action cannot be undone.", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No)
|
||||
if confirm == QMessageBox.StandardButton.Yes:
|
||||
if state.dl_session:
|
||||
state.dl_session.remove_torrent(magnetdl)
|
||||
if hasattr(magnetdl, 'stop'):
|
||||
magnetdl.stop()
|
||||
elif state.dl_session:
|
||||
try:
|
||||
state.dl_session.remove_torrent(magnetdl)
|
||||
except Exception:
|
||||
pass
|
||||
if download_path and os.path.exists(download_path):
|
||||
try:
|
||||
if os.path.isfile(download_path):
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
from typing import Optional
|
||||
from core.utils.data.state import state
|
||||
from core.utils.logging.logs import consoleLog, add_download_log
|
||||
|
||||
from .handle import DirectDownloadHandle
|
||||
from .utils import (
|
||||
sanitize_filename,
|
||||
extract_filename_from_url,
|
||||
detect_filename_from_headers,
|
||||
)
|
||||
|
||||
def add_direct_download(url: str, title: str, dl_path: Optional[str] = None):
|
||||
|
||||
if dl_path is None:
|
||||
dl_path = state.download_path
|
||||
|
||||
if url in state.active_downloads:
|
||||
consoleLog(f"Download already active: {title}")
|
||||
return
|
||||
|
||||
filename = (
|
||||
detect_filename_from_headers(url, DirectDownloadHandle.USER_AGENT)
|
||||
or extract_filename_from_url(url)
|
||||
or sanitize_filename(title) + ".zip"
|
||||
)
|
||||
|
||||
handle = DirectDownloadHandle(url, filename, dl_path)
|
||||
state.active_downloads[url] = handle
|
||||
add_download_log(title, url, "", False)
|
||||
handle.start()
|
||||
consoleLog(f"Started direct download: {filename}")
|
||||
@@ -0,0 +1,443 @@
|
||||
import os
|
||||
import threading
|
||||
import requests
|
||||
import json
|
||||
import hashlib
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Optional
|
||||
import libtorrent as lt
|
||||
|
||||
from core.utils.logging.logs import consoleLog, update_download_completed
|
||||
from core.utils.data.state import state
|
||||
from .status import DirectDownloadStatus, ChunkSpec
|
||||
from .utils import format_size
|
||||
|
||||
class SimpleThrottler:
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._last_time = time.monotonic()
|
||||
self._allowance = 0.0
|
||||
|
||||
def throttle(self, bytes_count: int, limit_kbps: int):
|
||||
if limit_kbps <= 0:
|
||||
return
|
||||
|
||||
limit_bps = limit_kbps * 1024
|
||||
wait_time = 0
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_time
|
||||
self._last_time = now
|
||||
|
||||
self._allowance += elapsed * limit_bps
|
||||
if self._allowance > 2 * limit_bps:
|
||||
self._allowance = 2 * limit_bps
|
||||
|
||||
self._allowance -= bytes_count
|
||||
|
||||
if self._allowance < 0:
|
||||
wait_time = -self._allowance / limit_bps
|
||||
self._allowance = 0
|
||||
|
||||
if wait_time > 0:
|
||||
time.sleep(min(wait_time, 1.0))
|
||||
|
||||
_throttler = SimpleThrottler()
|
||||
|
||||
class DirectDownloadHandle:
|
||||
STREAM_BLOCK_SIZE = 1 << 17 # 128 KiB per read
|
||||
MAX_RETRIES = 5
|
||||
RETRY_BACKOFF_BASE = 2.0
|
||||
NUM_THREADS = 16
|
||||
MIN_CHUNK_SIZE = 1 << 20 # 1 MiB minimum per chunk
|
||||
REQUEST_TIMEOUT = (15, 60) # (connect, read) timeouts
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
def __init__(self, url: str, name: str, save_path: str):
|
||||
self.url = url
|
||||
self._name = name
|
||||
self._save_path = save_path
|
||||
self._file_path = os.path.join(save_path, name)
|
||||
self._status = DirectDownloadStatus(name, save_path, 0)
|
||||
self._stop_event = threading.Event()
|
||||
self._pause_event = threading.Event() # SET = paused
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session: Optional[requests.Session] = None
|
||||
self._supports_range = False
|
||||
|
||||
state_dir = os.path.join(state.settings_path, "direct_downloads")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
url_hash = hashlib.sha256(url.encode()).hexdigest()
|
||||
self._state_file = os.path.join(state_dir, f"{url_hash}.json")
|
||||
|
||||
# Load state to initialize progress
|
||||
state_data = self._load_state()
|
||||
if state_data:
|
||||
total_wanted = state_data.get("total_wanted", 0)
|
||||
chunks_progress = state_data.get("chunks", {})
|
||||
chunks_progress = {int(k): int(v) for k, v in chunks_progress.items()}
|
||||
self._status.initialize_progress(total_wanted, chunks_progress)
|
||||
|
||||
|
||||
def status(self) -> DirectDownloadStatus:
|
||||
return self._status
|
||||
|
||||
def pause(self):
|
||||
self._status.paused = True
|
||||
self._pause_event.set()
|
||||
|
||||
def resume(self):
|
||||
self._status.paused = False
|
||||
self._pause_event.clear()
|
||||
|
||||
def set_flags(self, flags):
|
||||
if flags & lt.torrent_flags.auto_managed:
|
||||
self._status.auto_managed = True
|
||||
|
||||
def unset_flags(self, flags):
|
||||
if flags & lt.torrent_flags.auto_managed:
|
||||
self._status.auto_managed = False
|
||||
|
||||
def save_path(self) -> str:
|
||||
return self._save_path
|
||||
|
||||
def stop(self):
|
||||
self._stop_event.set()
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
|
||||
def _load_state(self) -> dict:
|
||||
if os.path.exists(self._state_file):
|
||||
try:
|
||||
with open(self._state_file, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
consoleLog(f"Failed to load state for {self._name}: {e}")
|
||||
return {}
|
||||
|
||||
def _save_state(self, chunks_done: dict[int, int]):
|
||||
try:
|
||||
with open(self._state_file, "w") as f:
|
||||
json.dump({
|
||||
"url": self.url,
|
||||
"name": self._name,
|
||||
"save_path": self._save_path,
|
||||
"total_wanted": self._status.total_wanted,
|
||||
"chunks": chunks_done
|
||||
}, f)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
def _clear_state(self):
|
||||
if os.path.exists(self._state_file):
|
||||
try:
|
||||
os.remove(self._state_file)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(
|
||||
target=self._download_orchestrator, name=f"dl-{self._name}", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _build_session(self) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": self.USER_AGENT})
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
max_retries=0,
|
||||
pool_connections=self.NUM_THREADS,
|
||||
pool_maxsize=self.NUM_THREADS + 2,
|
||||
)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
def _probe_url(self, session: requests.Session) -> tuple[int, bool]:
|
||||
resp = session.head(self.url, allow_redirects=True, timeout=self.REQUEST_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
|
||||
total_size = int(resp.headers.get("content-length", 0))
|
||||
supports_range = (total_size > 0)
|
||||
|
||||
# Double-check range support with a small range request
|
||||
if supports_range:
|
||||
try:
|
||||
test = session.get(
|
||||
self.url,
|
||||
headers={"Range": "bytes=0-0"},
|
||||
timeout=self.REQUEST_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
supports_range = test.status_code == 206
|
||||
test.close()
|
||||
except Exception:
|
||||
supports_range = False
|
||||
|
||||
return total_size, supports_range
|
||||
|
||||
def _download_orchestrator(self):
|
||||
try:
|
||||
self._session = self._build_session()
|
||||
total_size, self._supports_range = self._probe_url(self._session)
|
||||
self._status.total_wanted = total_size
|
||||
|
||||
os.makedirs(self._save_path, exist_ok=True)
|
||||
|
||||
use_multithreaded = (
|
||||
self._supports_range
|
||||
and total_size > self.MIN_CHUNK_SIZE * 2
|
||||
)
|
||||
|
||||
if use_multithreaded:
|
||||
consoleLog(
|
||||
f"Multi-threaded download ({self.NUM_THREADS} threads): {self._name} "
|
||||
f"({format_size(total_size)})"
|
||||
)
|
||||
self._preallocate_file(total_size)
|
||||
self._multithreaded_download(total_size)
|
||||
else:
|
||||
reason = "no range support" if not self._supports_range else "file too small"
|
||||
consoleLog(f"Single-threaded download ({reason}): {self._name}")
|
||||
self._single_threaded_download()
|
||||
|
||||
if not self._stop_event.is_set() and self._status.error is None:
|
||||
self._verify_download(total_size)
|
||||
self._status.mark_completed()
|
||||
update_download_completed(self.url, True)
|
||||
self._clear_state()
|
||||
consoleLog(f"✓ Finished downloading {self._name}")
|
||||
|
||||
except Exception as e:
|
||||
self._status.mark_error(str(e))
|
||||
consoleLog(f"✗ Download failed for {self._name}: {e}")
|
||||
finally:
|
||||
if self._session:
|
||||
self._session.close()
|
||||
|
||||
|
||||
def _preallocate_file(self, total_size: int):
|
||||
if os.path.exists(self._file_path) and os.path.getsize(self._file_path) == total_size:
|
||||
return
|
||||
with open(self._file_path, "wb") as f:
|
||||
f.truncate(total_size)
|
||||
|
||||
def _compute_chunks(self, total_size: int) -> list[ChunkSpec]:
|
||||
num_chunks = min(self.NUM_THREADS, max(1, total_size // self.MIN_CHUNK_SIZE))
|
||||
chunk_size = total_size // num_chunks
|
||||
chunks = []
|
||||
for i in range(num_chunks):
|
||||
start = i * chunk_size
|
||||
end = (i + 1) * chunk_size - 1 if i < num_chunks - 1 else total_size - 1
|
||||
chunks.append(ChunkSpec(chunk_id=i, start=start, end=end))
|
||||
return chunks
|
||||
|
||||
def _multithreaded_download(self, total_size: int):
|
||||
chunks = self._compute_chunks(total_size)
|
||||
state_data = self._load_state()
|
||||
chunks_progress = state_data.get("chunks", {})
|
||||
|
||||
chunks_progress = {int(k): int(v) for k, v in chunks_progress.items()}
|
||||
|
||||
# Initialize progress in status
|
||||
for chunk_id, bytes_done in chunks_progress.items():
|
||||
self._status.update_chunk_progress(chunk_id, bytes_done)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=len(chunks), thread_name_prefix="dl-chunk"
|
||||
) as executor:
|
||||
futures = {
|
||||
executor.submit(self._download_chunk_with_retry, chunk, chunks_progress.get(chunk.chunk_id, 0)): chunk
|
||||
for chunk in chunks
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
chunk = futures[future]
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
consoleLog(
|
||||
f"Chunk {chunk.chunk_id} ({chunk.start}-{chunk.end}) "
|
||||
f"failed permanently: {e}"
|
||||
)
|
||||
self._status.mark_error(
|
||||
f"Chunk {chunk.chunk_id} failed: {e}"
|
||||
)
|
||||
self._stop_event.set()
|
||||
|
||||
def _download_chunk_with_retry(self, chunk: ChunkSpec, initial_bytes: int):
|
||||
bytes_written = initial_bytes
|
||||
|
||||
for attempt in range(1, self.MAX_RETRIES + 1):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
current_start = chunk.start + bytes_written
|
||||
if current_start > chunk.end:
|
||||
return
|
||||
|
||||
try:
|
||||
# _download_range returns new bytes written
|
||||
bytes_written += self._download_range(
|
||||
chunk.chunk_id, current_start, chunk.end, bytes_written
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if attempt < self.MAX_RETRIES:
|
||||
wait = self.RETRY_BACKOFF_BASE ** attempt
|
||||
consoleLog(
|
||||
f"Chunk {chunk.chunk_id} attempt {attempt} failed: {e}. "
|
||||
f"Retrying in {wait:.0f}s (resuming from byte {current_start})..."
|
||||
)
|
||||
if self._stop_event.wait(timeout=wait):
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Chunk {chunk.chunk_id} failed after {self.MAX_RETRIES} attempts: {e}"
|
||||
) from e
|
||||
|
||||
def _download_range(
|
||||
self, chunk_id: int, start: int, end: int, prior_bytes: int
|
||||
) -> int:
|
||||
|
||||
headers = {"Range": f"bytes={start}-{end}"}
|
||||
new_bytes = 0
|
||||
|
||||
with self._session.get(
|
||||
self.url, headers=headers, stream=True, timeout=self.REQUEST_TIMEOUT
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
if resp.status_code not in (200, 206):
|
||||
raise RuntimeError(f"Unexpected status {resp.status_code}")
|
||||
|
||||
with open(self._file_path, "r+b") as f:
|
||||
f.seek(start)
|
||||
for block in resp.iter_content(chunk_size=self.STREAM_BLOCK_SIZE):
|
||||
if self._stop_event.is_set():
|
||||
return new_bytes
|
||||
|
||||
self._wait_if_paused()
|
||||
if self._stop_event.is_set():
|
||||
return new_bytes
|
||||
|
||||
_throttler.throttle(len(block), state.down_speed_limit)
|
||||
|
||||
f.write(block)
|
||||
new_bytes += len(block)
|
||||
self._status.update_chunk_progress(
|
||||
chunk_id, prior_bytes + new_bytes
|
||||
)
|
||||
|
||||
|
||||
if new_bytes % (1024 * 1024) < len(block):
|
||||
|
||||
with self._status._lock:
|
||||
self._save_state(self._status._chunk_bytes)
|
||||
|
||||
with self._status._lock:
|
||||
self._save_state(self._status._chunk_bytes)
|
||||
|
||||
return new_bytes
|
||||
|
||||
|
||||
def _single_threaded_download(self):
|
||||
chunk_id = 0
|
||||
bytes_written = 0
|
||||
|
||||
if os.path.exists(self._file_path):
|
||||
bytes_written = os.path.getsize(self._file_path)
|
||||
if bytes_written > 0:
|
||||
self._status.update_chunk_progress(chunk_id, bytes_written)
|
||||
|
||||
for attempt in range(1, self.MAX_RETRIES + 1):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
try:
|
||||
headers = {}
|
||||
mode = "wb"
|
||||
|
||||
if bytes_written > 0 and self._supports_range:
|
||||
headers["Range"] = f"bytes={bytes_written}-"
|
||||
mode = "r+b"
|
||||
elif bytes_written > 0:
|
||||
bytes_written = 0
|
||||
mode = "wb"
|
||||
|
||||
with self._session.get(
|
||||
self.url, headers=headers, stream=True, timeout=self.REQUEST_TIMEOUT
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
|
||||
if self._status.total_wanted == 0:
|
||||
content_length = int(resp.headers.get("content-length", 0))
|
||||
self._status.total_wanted = content_length + bytes_written
|
||||
|
||||
with open(self._file_path, mode) as f:
|
||||
if mode == "r+b":
|
||||
f.seek(bytes_written)
|
||||
|
||||
for block in resp.iter_content(
|
||||
chunk_size=self.STREAM_BLOCK_SIZE
|
||||
):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
self._wait_if_paused()
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
_throttler.throttle(len(block), state.down_speed_limit)
|
||||
|
||||
f.write(block)
|
||||
bytes_written += len(block)
|
||||
self._status.update_chunk_progress(chunk_id, bytes_written)
|
||||
|
||||
# Periodic state save (single thread, chunk_id=0)
|
||||
if bytes_written % (1024 * 1024) < len(block):
|
||||
self._save_state({0: bytes_written})
|
||||
|
||||
self._save_state({0: bytes_written})
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if attempt < self.MAX_RETRIES:
|
||||
wait = self.RETRY_BACKOFF_BASE ** attempt
|
||||
consoleLog(
|
||||
f"Download attempt {attempt} failed: {e}. "
|
||||
f"Retrying in {wait:.0f}s..."
|
||||
)
|
||||
if self._stop_event.wait(timeout=wait):
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Download failed after {self.MAX_RETRIES} attempts: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
def _wait_if_paused(self):
|
||||
while self._pause_event.is_set():
|
||||
if self._stop_event.wait(timeout=0.5):
|
||||
return
|
||||
|
||||
def _verify_download(self, expected_size: int):
|
||||
if expected_size <= 0:
|
||||
return
|
||||
actual_size = os.path.getsize(self._file_path)
|
||||
if actual_size != expected_size:
|
||||
raise RuntimeError(
|
||||
f"Size mismatch: expected {format_size(expected_size)}, "
|
||||
f"got {format_size(actual_size)}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
import libtorrent as lt
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkSpec:
|
||||
chunk_id: int
|
||||
start: int
|
||||
end: int # inclusive
|
||||
|
||||
|
||||
class DirectDownloadStatus:
|
||||
|
||||
def __init__(self, name: str, save_path: str, total_size: int = 0):
|
||||
self.name = name
|
||||
self.save_path = save_path
|
||||
self.total_wanted = total_size
|
||||
self.paused = False
|
||||
self.auto_managed = True
|
||||
self.has_metadata = True
|
||||
self.state = lt.torrent_status.downloading
|
||||
self.error: Optional[str] = None
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._total_wanted_done = 0
|
||||
self._progress = 0.0
|
||||
self._download_rate = 0
|
||||
self._upload_rate = 0
|
||||
|
||||
self._chunk_bytes: dict[int, int] = {}
|
||||
self._speed_window: list[tuple[float, int]] = []
|
||||
self._speed_window_size = 3.0
|
||||
|
||||
@property
|
||||
def total_wanted_done(self) -> int:
|
||||
with self._lock:
|
||||
return self._total_wanted_done
|
||||
|
||||
@property
|
||||
def progress(self) -> float:
|
||||
with self._lock:
|
||||
return self._progress
|
||||
|
||||
@property
|
||||
def download_rate(self) -> int:
|
||||
with self._lock:
|
||||
return self._download_rate
|
||||
|
||||
@property
|
||||
def upload_rate(self) -> int:
|
||||
return 0
|
||||
|
||||
def update_chunk_progress(self, chunk_id: int, bytes_downloaded: int):
|
||||
with self._lock:
|
||||
self._chunk_bytes[chunk_id] = bytes_downloaded
|
||||
self._total_wanted_done = sum(self._chunk_bytes.values())
|
||||
if self.total_wanted > 0:
|
||||
self._progress = min(self._total_wanted_done / self.total_wanted, 1.0)
|
||||
|
||||
|
||||
now = time.monotonic()
|
||||
self._speed_window.append((now, self._total_wanted_done))
|
||||
|
||||
cutoff = now - self._speed_window_size
|
||||
self._speed_window = [
|
||||
(t, b) for t, b in self._speed_window if t >= cutoff
|
||||
]
|
||||
if len(self._speed_window) >= 2:
|
||||
oldest_time, oldest_bytes = self._speed_window[0]
|
||||
dt = now - oldest_time
|
||||
if dt > 0:
|
||||
self._download_rate = int(
|
||||
(self._total_wanted_done - oldest_bytes) / dt
|
||||
)
|
||||
|
||||
def initialize_progress(self, total_wanted: int, chunk_bytes: dict[int, int]):
|
||||
with self._lock:
|
||||
self.total_wanted = total_wanted
|
||||
self._chunk_bytes = chunk_bytes.copy()
|
||||
self._total_wanted_done = sum(self._chunk_bytes.values())
|
||||
if self.total_wanted > 0:
|
||||
self._progress = min(self._total_wanted_done / self.total_wanted, 1.0)
|
||||
|
||||
def mark_completed(self):
|
||||
with self._lock:
|
||||
self.state = lt.torrent_status.seeding
|
||||
self._progress = 1.0
|
||||
self._download_rate = 0
|
||||
|
||||
def mark_error(self, error: str):
|
||||
with self._lock:
|
||||
self.error = error
|
||||
self._download_rate = 0
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
import requests
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse, unquote
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
# Remove or replace dangerous characters
|
||||
keepchars = (" ", ".", "_", "-")
|
||||
cleaned = "".join(c for c in name if c.isalnum() or c in keepchars).strip()
|
||||
while " " in cleaned:
|
||||
cleaned = cleaned.replace(" ", " ")
|
||||
return cleaned or "download"
|
||||
|
||||
|
||||
def extract_filename_from_url(url: str) -> Optional[str]:
|
||||
parsed = urlparse(url)
|
||||
path = unquote(parsed.path)
|
||||
basename = os.path.basename(path)
|
||||
if basename and "." in basename and len(basename) < 256:
|
||||
return basename
|
||||
return None
|
||||
|
||||
|
||||
def detect_filename_from_headers(url: str, user_agent: str) -> Optional[str]:
|
||||
try:
|
||||
resp = requests.head(
|
||||
url,
|
||||
headers={"User-Agent": user_agent},
|
||||
allow_redirects=True,
|
||||
timeout=15,
|
||||
)
|
||||
cd = resp.headers.get("content-disposition", "")
|
||||
if "filename=" in cd:
|
||||
parts = cd.split("filename=")
|
||||
if len(parts) > 1:
|
||||
fname = parts[-1].strip().strip('"').strip("'")
|
||||
if fname:
|
||||
return fname
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def format_size(size_bytes: int) -> str:
|
||||
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
||||
if abs(size_bytes) < 1024:
|
||||
return f"{size_bytes:.1f} {unit}"
|
||||
size_bytes /= 1024
|
||||
return f"{size_bytes:.1f} PiB"
|
||||
@@ -19,8 +19,8 @@ def init_session():
|
||||
|
||||
|
||||
settings = {
|
||||
"upload_rate_limit": state.up_speed_limit,
|
||||
"download_rate_limit": state.down_speed_limit,
|
||||
"upload_rate_limit": state.up_speed_limit * 1024,
|
||||
"download_rate_limit": state.down_speed_limit * 1024,
|
||||
"enable_dht": True,
|
||||
"enable_lsd": True,
|
||||
"enable_upnp": True,
|
||||
@@ -149,8 +149,8 @@ def update_settings():
|
||||
return
|
||||
|
||||
settings = {
|
||||
"upload_rate_limit": state.up_speed_limit,
|
||||
"download_rate_limit": state.down_speed_limit,
|
||||
"upload_rate_limit": state.up_speed_limit * 1024,
|
||||
"download_rate_limit": state.down_speed_limit * 1024,
|
||||
"connections_limit": state.max_connections,
|
||||
"active_downloads": state.max_downloads
|
||||
}
|
||||
|
||||
@@ -16,8 +16,13 @@ def check_completed(downloads, resume):
|
||||
consoleLog(f"Found unfinished download: {download.title}")
|
||||
if resume == True:
|
||||
dl_dir = os.path.dirname(download.path)
|
||||
run_download_direct(download.magnet_uri, dl_dir)
|
||||
consoleLog(f"Resuming {download.title}")
|
||||
if download.magnet_uri:
|
||||
run_download_direct(download.magnet_uri, dl_dir, download.title)
|
||||
consoleLog(f"Resuming Magnet: {download.title}")
|
||||
elif download.url:
|
||||
from core.network.direct_download import add_direct_download
|
||||
add_direct_download(download.url, download.title, dl_dir)
|
||||
consoleLog(f"Resuming Direct Download: {download.title}")
|
||||
|
||||
def check_downloads(downloads):
|
||||
for download in downloads:
|
||||
|
||||
@@ -6,11 +6,10 @@ from core.utils.general.wrappers import run_thread
|
||||
from core.utils.logging.logs import add_download_log
|
||||
from core.network.libtorrent_int import add_seed
|
||||
from core.utils.data.state import state
|
||||
from urllib.parse import urlparse, unquote
|
||||
from core.network.direct_download import add_direct_download
|
||||
import threading
|
||||
import re
|
||||
import requests
|
||||
import random
|
||||
|
||||
|
||||
|
||||
|
||||
def download_selected(items: list[QTableWidgetItem]):
|
||||
@@ -31,37 +30,6 @@ def download_selected(items: list[QTableWidgetItem]):
|
||||
consoleLog(f"Downloading {post.get('title', 'Unknown')}")
|
||||
run_thread(threading.Thread(target=run_download, args=(post,)))
|
||||
|
||||
def get_direct_filename(url: str, headers) -> str:
|
||||
cd = headers.get('content-disposition', '')
|
||||
if cd:
|
||||
match = re.search(r'filename\*=UTF-8\'\'(.+)', cd) # RFC 5987 encoded
|
||||
if match:
|
||||
return unquote(match.group(1))
|
||||
match = re.search(r'filename="?([^";\n]+)"?', cd)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
|
||||
path = urlparse(url).path
|
||||
name = path.split('/')[-1]
|
||||
if name:
|
||||
return unquote(name)
|
||||
|
||||
return f"download{random.randint(1,9999)}" #avoid collision
|
||||
|
||||
def filedownload(url):
|
||||
try:
|
||||
with requests.get(url, stream=True, timeout=30) as r:
|
||||
r.raise_for_status()
|
||||
name = get_direct_filename(url, r.headers)
|
||||
|
||||
with open(state.download_path + f"/{name}", 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
consoleLog(f"Finished downloading {name}")
|
||||
except Exception as e:
|
||||
consoleLog(f"Error downloading: {e}")
|
||||
|
||||
def run_download(post):
|
||||
linkfunc = state.trackers[state.currenttracker]["linkFunc"]
|
||||
ismagnet = state.trackers[state.currenttracker]["isMagnet"]
|
||||
@@ -71,12 +39,12 @@ def run_download(post):
|
||||
add_magnet(link)
|
||||
add_download_log(post.get("title", "Unknown"), "", link, False)
|
||||
else:
|
||||
filedownload(link)
|
||||
add_direct_download(link, post.get("title", "Unknown"))
|
||||
|
||||
def run_download_direct(magnet_uri, dl_path=None):
|
||||
consoleLog(f"Direct download: {magnet_uri[:60]}")
|
||||
def run_download_direct(magnet_uri, dl_path=None, title="Direct Download"):
|
||||
consoleLog(f"Magnet: {title}")
|
||||
add_magnet(magnet_uri, dl_path)
|
||||
add_download_log("Direct Download", "", magnet_uri, False)
|
||||
add_download_log(title, "", magnet_uri, False)
|
||||
|
||||
def seed_magnet(magnet_uri, file_path):
|
||||
consoleLog(f"Seeding: {magnet_uri[:60]}")
|
||||
|
||||
Reference in New Issue
Block a user