mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-04 09:59:41 +02:00
refactor: remove core/ folder
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
from utils.logging.logs import consoleLog, add_download_log
|
||||
from utils.data.state import state
|
||||
from typing import Optional
|
||||
|
||||
|
||||
from .handle import DirectDownloadHandle
|
||||
from .utils import (
|
||||
sanitize_filename,
|
||||
extract_filename_from_url,
|
||||
detect_filename_from_headers,
|
||||
)
|
||||
|
||||
def add_direct_download(url: str, title: str, dl_path: Optional[str] = None, headers: Optional[dict] = None, single_threaded: bool = False):
|
||||
|
||||
if dl_path is None:
|
||||
dl_path = state.download_path
|
||||
|
||||
if url in state.active_downloads:
|
||||
consoleLog(f"Download already active: {title}")
|
||||
return
|
||||
|
||||
filename = (
|
||||
detect_filename_from_headers(url, DirectDownloadHandle.USER_AGENT)
|
||||
or extract_filename_from_url(url)
|
||||
or sanitize_filename(title) + ".zip"
|
||||
)
|
||||
|
||||
handle = DirectDownloadHandle(url, filename, dl_path, headers, single_threaded)
|
||||
state.active_downloads[url] = handle
|
||||
add_download_log(title, url, "", False)
|
||||
handle.start()
|
||||
consoleLog(f"Started direct download: {filename}")
|
||||
@@ -0,0 +1,447 @@
|
||||
from network.direct_download.status import DirectDownloadStatus, ChunkSpec
|
||||
from utils.logging.logs import consoleLog, update_download_completed
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from network.direct_download.utils import format_size
|
||||
from utils.data.state import state
|
||||
from typing import Optional
|
||||
import libtorrent as lt
|
||||
import threading
|
||||
import requests
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
class SimpleThrottler:
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._last_time = time.monotonic()
|
||||
self._allowance = 0.0
|
||||
|
||||
def throttle(self, bytes_count: int, limit_kbps: int):
|
||||
if limit_kbps <= 0:
|
||||
return
|
||||
|
||||
limit_bps = limit_kbps * 1024
|
||||
wait_time = 0
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
elapsed = now - self._last_time
|
||||
self._last_time = now
|
||||
|
||||
self._allowance += elapsed * limit_bps
|
||||
if self._allowance > 2 * limit_bps:
|
||||
self._allowance = 2 * limit_bps
|
||||
|
||||
self._allowance -= bytes_count
|
||||
|
||||
if self._allowance < 0:
|
||||
wait_time = -self._allowance / limit_bps
|
||||
self._allowance = 0
|
||||
|
||||
if wait_time > 0:
|
||||
time.sleep(min(wait_time, 1.0))
|
||||
|
||||
_throttler = SimpleThrottler()
|
||||
|
||||
class DirectDownloadHandle:
|
||||
STREAM_BLOCK_SIZE = 1 << 17 # 128 KiB per read
|
||||
MAX_RETRIES = 5
|
||||
RETRY_BACKOFF_BASE = 2.0
|
||||
NUM_THREADS = 16
|
||||
MIN_CHUNK_SIZE = 1 << 20 # 1 MiB minimum per chunk
|
||||
REQUEST_TIMEOUT = (15, 60) # (connect, read) timeouts
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
def __init__(self, url: str, name: str, save_path: str, headers: Optional[dict] = None, single_threaded: bool = False):
|
||||
self.url = url
|
||||
self._name = name
|
||||
self._save_path = save_path
|
||||
self._file_path = os.path.join(save_path, name)
|
||||
self._status = DirectDownloadStatus(name, save_path, 0)
|
||||
self._stop_event = threading.Event()
|
||||
self._pause_event = threading.Event() # SET = paused
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._session: Optional[requests.Session] = None
|
||||
self._supports_range = False
|
||||
self._custom_headers = headers
|
||||
self._single_threaded = single_threaded
|
||||
|
||||
state_dir = os.path.join(state.settings_path, "direct_downloads")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
url_hash = hashlib.sha256(url.encode()).hexdigest()
|
||||
self._state_file = os.path.join(state_dir, f"{url_hash}.json")
|
||||
|
||||
# Load state to initialize progress
|
||||
state_data = self._load_state()
|
||||
if state_data:
|
||||
total_wanted = state_data.get("total_wanted", 0)
|
||||
chunks_progress = state_data.get("chunks", {})
|
||||
chunks_progress = {int(k): int(v) for k, v in chunks_progress.items()}
|
||||
self._status.initialize_progress(total_wanted, chunks_progress)
|
||||
|
||||
|
||||
def status(self) -> DirectDownloadStatus:
|
||||
return self._status
|
||||
|
||||
def pause(self):
|
||||
self._status.paused = True
|
||||
self._pause_event.set()
|
||||
|
||||
def resume(self):
|
||||
self._status.paused = False
|
||||
self._pause_event.clear()
|
||||
|
||||
def set_flags(self, flags):
|
||||
if flags & lt.torrent_flags.auto_managed:
|
||||
self._status.auto_managed = True
|
||||
|
||||
def unset_flags(self, flags):
|
||||
if flags & lt.torrent_flags.auto_managed:
|
||||
self._status.auto_managed = False
|
||||
|
||||
def save_path(self) -> str:
|
||||
return self._save_path
|
||||
|
||||
def stop(self):
|
||||
self._stop_event.set()
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
|
||||
def _load_state(self) -> dict:
|
||||
if os.path.exists(self._state_file):
|
||||
try:
|
||||
with open(self._state_file, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
consoleLog(f"Failed to load state for {self._name}: {e}")
|
||||
return {}
|
||||
|
||||
def _save_state(self, chunks_done: dict[int, int]):
|
||||
try:
|
||||
with open(self._state_file, "w") as f:
|
||||
json.dump({
|
||||
"url": self.url,
|
||||
"name": self._name,
|
||||
"save_path": self._save_path,
|
||||
"total_wanted": self._status.total_wanted,
|
||||
"chunks": chunks_done
|
||||
}, f)
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while saving download state: {e}")
|
||||
|
||||
def _clear_state(self):
|
||||
if os.path.exists(self._state_file):
|
||||
try:
|
||||
os.remove(self._state_file)
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while removing download state file: {e}")
|
||||
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(
|
||||
target=self._download_orchestrator, name=f"dl-{self._name}", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _build_session(self) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": self.USER_AGENT})
|
||||
if self._custom_headers is not None:
|
||||
session.headers.update(self._custom_headers)
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
max_retries=0,
|
||||
pool_connections=self.NUM_THREADS,
|
||||
pool_maxsize=self.NUM_THREADS + 2,
|
||||
)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
def _probe_url(self, session: requests.Session) -> tuple[int, bool]:
|
||||
resp = session.head(self.url, allow_redirects=True, timeout=self.REQUEST_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
|
||||
total_size = int(resp.headers.get("content-length", 0))
|
||||
supports_range = (total_size > 0)
|
||||
|
||||
# Double-check range support with a small range request
|
||||
if supports_range:
|
||||
try:
|
||||
test = session.get(
|
||||
self.url,
|
||||
headers={"Range": "bytes=0-0"},
|
||||
timeout=self.REQUEST_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
supports_range = test.status_code == 206
|
||||
test.close()
|
||||
except Exception:
|
||||
supports_range = False
|
||||
|
||||
return total_size, supports_range
|
||||
|
||||
def _download_orchestrator(self):
|
||||
try:
|
||||
self._session = self._build_session()
|
||||
total_size, self._supports_range = self._probe_url(self._session)
|
||||
self._status.total_wanted = total_size
|
||||
|
||||
os.makedirs(self._save_path, exist_ok=True)
|
||||
|
||||
use_multithreaded = (
|
||||
self._supports_range
|
||||
and total_size > self.MIN_CHUNK_SIZE * 2
|
||||
and not self._single_threaded
|
||||
)
|
||||
|
||||
if use_multithreaded:
|
||||
consoleLog(
|
||||
f"Multi-threaded download ({self.NUM_THREADS} threads): {self._name} "
|
||||
f"({format_size(total_size)})"
|
||||
)
|
||||
self._preallocate_file(total_size)
|
||||
self._multithreaded_download(total_size)
|
||||
else:
|
||||
reason = "no range support" if not self._supports_range else "file too small or single-threaded mode"
|
||||
consoleLog(f"Single-threaded download ({reason}): {self._name}")
|
||||
self._single_threaded_download()
|
||||
|
||||
if not self._stop_event.is_set() and self._status.error is None:
|
||||
self._verify_download(total_size)
|
||||
self._status.mark_completed()
|
||||
update_download_completed(self.url, True)
|
||||
self._clear_state()
|
||||
consoleLog(f"Finished downloading {self._name}")
|
||||
|
||||
except Exception as e:
|
||||
self._status.mark_error(str(e))
|
||||
consoleLog(f"Download failed for {self._name}: {e}")
|
||||
finally:
|
||||
if self._session:
|
||||
self._session.close()
|
||||
|
||||
|
||||
def _preallocate_file(self, total_size: int):
|
||||
if os.path.exists(self._file_path) and os.path.getsize(self._file_path) == total_size:
|
||||
return
|
||||
with open(self._file_path, "wb") as f:
|
||||
f.truncate(total_size)
|
||||
|
||||
def _compute_chunks(self, total_size: int) -> list[ChunkSpec]:
|
||||
num_chunks = min(self.NUM_THREADS, max(1, total_size // self.MIN_CHUNK_SIZE))
|
||||
chunk_size = total_size // num_chunks
|
||||
chunks = []
|
||||
for i in range(num_chunks):
|
||||
start = i * chunk_size
|
||||
end = (i + 1) * chunk_size - 1 if i < num_chunks - 1 else total_size - 1
|
||||
chunks.append(ChunkSpec(chunk_id=i, start=start, end=end))
|
||||
return chunks
|
||||
|
||||
def _multithreaded_download(self, total_size: int):
|
||||
chunks = self._compute_chunks(total_size)
|
||||
state_data = self._load_state()
|
||||
chunks_progress = state_data.get("chunks", {})
|
||||
|
||||
chunks_progress = {int(k): int(v) for k, v in chunks_progress.items()}
|
||||
|
||||
# Initialize progress in status
|
||||
for chunk_id, bytes_done in chunks_progress.items():
|
||||
self._status.update_chunk_progress(chunk_id, bytes_done)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=len(chunks), thread_name_prefix="dl-chunk"
|
||||
) as executor:
|
||||
futures = {
|
||||
executor.submit(self._download_chunk_with_retry, chunk, chunks_progress.get(chunk.chunk_id, 0)): chunk
|
||||
for chunk in chunks
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
chunk = futures[future]
|
||||
try:
|
||||
future.result()
|
||||
except Exception as e:
|
||||
consoleLog(
|
||||
f"Chunk {chunk.chunk_id} ({chunk.start}-{chunk.end}) "
|
||||
f"failed permanently: {e}"
|
||||
)
|
||||
self._status.mark_error(
|
||||
f"Chunk {chunk.chunk_id} failed: {e}"
|
||||
)
|
||||
self._stop_event.set()
|
||||
|
||||
def _download_chunk_with_retry(self, chunk: ChunkSpec, initial_bytes: int):
|
||||
bytes_written = initial_bytes
|
||||
|
||||
for attempt in range(1, self.MAX_RETRIES + 1):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
current_start = chunk.start + bytes_written
|
||||
if current_start > chunk.end:
|
||||
return
|
||||
|
||||
try:
|
||||
# _download_range returns new bytes written
|
||||
bytes_written += self._download_range(
|
||||
chunk.chunk_id, current_start, chunk.end, bytes_written
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if attempt < self.MAX_RETRIES:
|
||||
wait = self.RETRY_BACKOFF_BASE ** attempt
|
||||
consoleLog(
|
||||
f"Chunk {chunk.chunk_id} attempt {attempt} failed: {e}. "
|
||||
f"Retrying in {wait:.0f}s (resuming from byte {current_start})..."
|
||||
)
|
||||
if self._stop_event.wait(timeout=wait):
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Chunk {chunk.chunk_id} failed after {self.MAX_RETRIES} attempts: {e}"
|
||||
) from e
|
||||
|
||||
def _download_range(
|
||||
self, chunk_id: int, start: int, end: int, prior_bytes: int
|
||||
) -> int:
|
||||
|
||||
headers = {"Range": f"bytes={start}-{end}"}
|
||||
new_bytes = 0
|
||||
|
||||
with self._session.get(
|
||||
self.url, headers=headers, stream=True, timeout=self.REQUEST_TIMEOUT
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
if resp.status_code not in (200, 206):
|
||||
raise RuntimeError(f"Unexpected status {resp.status_code}")
|
||||
|
||||
with open(self._file_path, "r+b") as f:
|
||||
f.seek(start)
|
||||
for block in resp.iter_content(chunk_size=self.STREAM_BLOCK_SIZE):
|
||||
if self._stop_event.is_set():
|
||||
return new_bytes
|
||||
|
||||
self._wait_if_paused()
|
||||
if self._stop_event.is_set():
|
||||
return new_bytes
|
||||
|
||||
_throttler.throttle(len(block), state.down_speed_limit)
|
||||
|
||||
f.write(block)
|
||||
new_bytes += len(block)
|
||||
self._status.update_chunk_progress(
|
||||
chunk_id, prior_bytes + new_bytes
|
||||
)
|
||||
|
||||
|
||||
if new_bytes % (1024 * 1024) < len(block):
|
||||
|
||||
with self._status._lock:
|
||||
self._save_state(self._status._chunk_bytes)
|
||||
|
||||
with self._status._lock:
|
||||
self._save_state(self._status._chunk_bytes)
|
||||
|
||||
return new_bytes
|
||||
|
||||
|
||||
def _single_threaded_download(self):
|
||||
chunk_id = 0
|
||||
bytes_written = 0
|
||||
|
||||
if os.path.exists(self._file_path):
|
||||
bytes_written = os.path.getsize(self._file_path)
|
||||
if bytes_written > 0:
|
||||
self._status.update_chunk_progress(chunk_id, bytes_written)
|
||||
|
||||
for attempt in range(1, self.MAX_RETRIES + 1):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
try:
|
||||
headers = {}
|
||||
mode = "wb"
|
||||
|
||||
if bytes_written > 0 and self._supports_range:
|
||||
headers["Range"] = f"bytes={bytes_written}-"
|
||||
mode = "r+b"
|
||||
elif bytes_written > 0:
|
||||
bytes_written = 0
|
||||
mode = "wb"
|
||||
|
||||
with self._session.get(
|
||||
self.url, headers=headers, stream=True, timeout=self.REQUEST_TIMEOUT
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
|
||||
if self._status.total_wanted == 0:
|
||||
content_length = int(resp.headers.get("content-length", 0))
|
||||
self._status.total_wanted = content_length + bytes_written
|
||||
|
||||
with open(self._file_path, mode) as f:
|
||||
if mode == "r+b":
|
||||
f.seek(bytes_written)
|
||||
|
||||
for block in resp.iter_content(
|
||||
chunk_size=self.STREAM_BLOCK_SIZE
|
||||
):
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
self._wait_if_paused()
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
|
||||
_throttler.throttle(len(block), state.down_speed_limit)
|
||||
|
||||
f.write(block)
|
||||
bytes_written += len(block)
|
||||
self._status.update_chunk_progress(chunk_id, bytes_written)
|
||||
|
||||
# Periodic state save (single thread, chunk_id=0)
|
||||
if bytes_written % (1024 * 1024) < len(block):
|
||||
self._save_state({0: bytes_written})
|
||||
|
||||
self._save_state({0: bytes_written})
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
if attempt < self.MAX_RETRIES:
|
||||
wait = self.RETRY_BACKOFF_BASE ** attempt
|
||||
consoleLog(
|
||||
f"Download attempt {attempt} failed: {e}. "
|
||||
f"Retrying in {wait:.0f}s..."
|
||||
)
|
||||
if self._stop_event.wait(timeout=wait):
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Download failed after {self.MAX_RETRIES} attempts: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
def _wait_if_paused(self):
|
||||
while self._pause_event.is_set():
|
||||
if self._stop_event.wait(timeout=0.5):
|
||||
return
|
||||
|
||||
def _verify_download(self, expected_size: int):
|
||||
if expected_size <= 0:
|
||||
return
|
||||
actual_size = os.path.getsize(self._file_path)
|
||||
if actual_size != expected_size:
|
||||
raise RuntimeError(
|
||||
f"Size mismatch: expected {format_size(expected_size)}, "
|
||||
f"got {format_size(actual_size)}"
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
import libtorrent as lt
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkSpec:
|
||||
chunk_id: int
|
||||
start: int
|
||||
end: int # inclusive
|
||||
|
||||
|
||||
class DirectDownloadStatus:
|
||||
|
||||
def __init__(self, name: str, save_path: str, total_size: int = 0):
|
||||
self.name = name
|
||||
self.save_path = save_path
|
||||
self.total_wanted = total_size
|
||||
self.paused = False
|
||||
self.auto_managed = True
|
||||
self.has_metadata = True
|
||||
self.state = lt.torrent_status.downloading
|
||||
self.error: Optional[str] = None
|
||||
|
||||
self._lock = threading.Lock()
|
||||
self._total_wanted_done = 0
|
||||
self._progress = 0.0
|
||||
self._download_rate = 0
|
||||
self._upload_rate = 0
|
||||
|
||||
self._chunk_bytes: dict[int, int] = {}
|
||||
self._speed_window: list[tuple[float, int]] = []
|
||||
self._speed_window_size = 3.0
|
||||
|
||||
@property
|
||||
def total_wanted_done(self) -> int:
|
||||
with self._lock:
|
||||
return self._total_wanted_done
|
||||
|
||||
@property
|
||||
def progress(self) -> float:
|
||||
with self._lock:
|
||||
return self._progress
|
||||
|
||||
@property
|
||||
def download_rate(self) -> int:
|
||||
with self._lock:
|
||||
return self._download_rate
|
||||
|
||||
@property
|
||||
def upload_rate(self) -> int:
|
||||
return 0
|
||||
|
||||
def update_chunk_progress(self, chunk_id: int, bytes_downloaded: int):
|
||||
with self._lock:
|
||||
self._chunk_bytes[chunk_id] = bytes_downloaded
|
||||
self._total_wanted_done = sum(self._chunk_bytes.values())
|
||||
if self.total_wanted > 0:
|
||||
self._progress = min(self._total_wanted_done / self.total_wanted, 1.0)
|
||||
|
||||
|
||||
now = time.monotonic()
|
||||
self._speed_window.append((now, self._total_wanted_done))
|
||||
|
||||
cutoff = now - self._speed_window_size
|
||||
self._speed_window = [
|
||||
(t, b) for t, b in self._speed_window if t >= cutoff
|
||||
]
|
||||
if len(self._speed_window) >= 2:
|
||||
oldest_time, oldest_bytes = self._speed_window[0]
|
||||
dt = now - oldest_time
|
||||
if dt > 0:
|
||||
self._download_rate = int(
|
||||
(self._total_wanted_done - oldest_bytes) / dt
|
||||
)
|
||||
|
||||
def initialize_progress(self, total_wanted: int, chunk_bytes: dict[int, int]):
|
||||
with self._lock:
|
||||
self.total_wanted = total_wanted
|
||||
self._chunk_bytes = chunk_bytes.copy()
|
||||
self._total_wanted_done = sum(self._chunk_bytes.values())
|
||||
if self.total_wanted > 0:
|
||||
self._progress = min(self._total_wanted_done / self.total_wanted, 1.0)
|
||||
|
||||
def mark_completed(self):
|
||||
with self._lock:
|
||||
self.state = lt.torrent_status.seeding
|
||||
self._progress = 1.0
|
||||
self._download_rate = 0
|
||||
|
||||
def mark_error(self, error: str):
|
||||
with self._lock:
|
||||
self.error = error
|
||||
self._download_rate = 0
|
||||
@@ -0,0 +1,49 @@
|
||||
from utils.logging.logs import consoleLog
|
||||
from urllib.parse import urlparse, unquote
|
||||
from typing import Optional
|
||||
import requests
|
||||
import os
|
||||
|
||||
def sanitize_filename(name: str) -> str:
|
||||
# Remove or replace dangerous characters
|
||||
keepchars = (" ", ".", "_", "-")
|
||||
cleaned = "".join(c for c in name if c.isalnum() or c in keepchars).strip()
|
||||
while " " in cleaned:
|
||||
cleaned = cleaned.replace(" ", " ")
|
||||
return cleaned or "download"
|
||||
|
||||
|
||||
def extract_filename_from_url(url: str) -> Optional[str]:
|
||||
parsed = urlparse(url)
|
||||
path = unquote(parsed.path)
|
||||
basename = os.path.basename(path)
|
||||
if basename and "." in basename and len(basename) < 256:
|
||||
return basename
|
||||
return None
|
||||
|
||||
|
||||
def detect_filename_from_headers(url: str, user_agent: str) -> Optional[str]:
|
||||
try:
|
||||
resp = requests.head(
|
||||
url,
|
||||
headers={"User-Agent": user_agent},
|
||||
allow_redirects=True,
|
||||
timeout=15,
|
||||
)
|
||||
cd = resp.headers.get("content-disposition", "")
|
||||
if "filename=" in cd:
|
||||
parts = cd.split("filename=")
|
||||
if len(parts) > 1:
|
||||
fname = parts[-1].strip().strip('"').strip("'")
|
||||
if fname:
|
||||
return fname
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while retrieving filename from headers: {e}")
|
||||
return None
|
||||
|
||||
def format_size(size_bytes: int) -> str:
|
||||
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
|
||||
if abs(size_bytes) < 1024:
|
||||
return f"{size_bytes:.1f} {unit}"
|
||||
size_bytes /= 1024
|
||||
return f"{size_bytes:.1f} PiB"
|
||||
@@ -0,0 +1,61 @@
|
||||
from utils.logging.logs import consoleLog
|
||||
from utils.data.state import state
|
||||
import psutil
|
||||
|
||||
def get_net_interfaces():
|
||||
addrs = psutil.net_if_addrs()
|
||||
for interface in addrs.keys():
|
||||
consoleLog(f"Found Interface: {interface}")
|
||||
return addrs.keys()
|
||||
|
||||
|
||||
def get_active_interfaces():
|
||||
addrs = psutil.net_if_addrs()
|
||||
stats = psutil.net_if_stats()
|
||||
active = []
|
||||
|
||||
for interface, addr_list in addrs.items():
|
||||
if interface not in stats:
|
||||
continue
|
||||
up = stats[interface].isup
|
||||
for addr in addr_list:
|
||||
if addr.family == 2: # ipv4
|
||||
ipv4 = addr.address
|
||||
if not ipv4.startswith("127.") and not ipv4.startswith("169.254") and up:
|
||||
active.append(interface)
|
||||
consoleLog(f"Found Active: {interface}")
|
||||
|
||||
return active
|
||||
|
||||
|
||||
def list_interfaces() -> None:
|
||||
consoleLog("Fetching Network Interfaces...")
|
||||
addrs = psutil.net_if_addrs()
|
||||
stats = psutil.net_if_stats()
|
||||
|
||||
for interface, addr_list in addrs.items():
|
||||
if interface not in stats:
|
||||
continue
|
||||
up = stats[interface].isup
|
||||
status = "INACTIVE"
|
||||
for addr in addr_list:
|
||||
if addr.family == 2: # ipv4
|
||||
ipv4 = addr.address
|
||||
if not ipv4.startswith("127.") and not ipv4.startswith("169.254") and up:
|
||||
status = "ACTIVE"
|
||||
consoleLog(f"Found Interface: {interface} [{status}]")
|
||||
|
||||
def init_interfaces():
|
||||
consoleLog("Initializing Interface variables...")
|
||||
addrs = psutil.net_if_addrs()
|
||||
state.interfaces = list(addrs.keys())
|
||||
state.active_interfaces = get_active_interfaces()
|
||||
|
||||
def get_interface_ip(interface_name):
|
||||
addrs = psutil.net_if_addrs()
|
||||
if interface_name in addrs:
|
||||
for addr in addrs[interface_name]:
|
||||
if addr.family == 2: # ipv4
|
||||
if not addr.address.startswith("127.") and not addr.address.startswith("169.254"):
|
||||
return addr.address
|
||||
return None
|
||||
@@ -0,0 +1,202 @@
|
||||
from network.interface import get_interface_ip
|
||||
from utils.general.wrappers import run_thread
|
||||
from utils.logging.logs import consoleLog
|
||||
from utils.data.state import state
|
||||
import libtorrent as lt
|
||||
import threading
|
||||
import platform
|
||||
import ctypes
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
loop_running = False
|
||||
|
||||
def get_free_space_mb(dirname):
|
||||
if platform.system() == 'Windows':
|
||||
free_bytes = ctypes.c_ulonglong(0)
|
||||
ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(dirname), None, None, ctypes.pointer(free_bytes))
|
||||
return free_bytes.value
|
||||
else:
|
||||
st = os.statvfs(dirname)
|
||||
return st.f_bavail * st.f_frsize
|
||||
|
||||
def init_session():
|
||||
if state.dl_session is not None:
|
||||
return
|
||||
|
||||
state.dl_session = lt.session()
|
||||
|
||||
|
||||
settings = {
|
||||
"upload_rate_limit": state.up_speed_limit * 1024,
|
||||
"download_rate_limit": state.down_speed_limit * 1024,
|
||||
"enable_dht": True,
|
||||
"enable_lsd": True,
|
||||
"enable_upnp": True,
|
||||
"enable_natpmp": True,
|
||||
"dht_bootstrap_nodes": "router.bittorrent.com:6881,dht.transmissionbt.com:6881",
|
||||
"connections_limit": state.max_connections,
|
||||
"active_downloads": state.max_downloads
|
||||
}
|
||||
|
||||
if state.bound_interface:
|
||||
interface_ip = get_interface_ip(state.bound_interface)
|
||||
if interface_ip:
|
||||
settings["outgoing_interfaces"] = interface_ip
|
||||
settings["listen_interfaces"] = f"{interface_ip}:6881"
|
||||
consoleLog(f"Binding to Interface IP: {interface_ip}")
|
||||
else:
|
||||
consoleLog(f"Skipping Binding, no Interface set. ({state.bound_interface})")
|
||||
|
||||
state.dl_session.apply_settings(settings)
|
||||
|
||||
consoleLog("Initialized Session")
|
||||
|
||||
|
||||
def add_download(magnet_uri):
|
||||
|
||||
if state.active_downloads is None:
|
||||
state.active_downloads = {}
|
||||
|
||||
init_session()
|
||||
|
||||
free_space = get_free_space_mb(state.download_path)
|
||||
|
||||
if magnet_uri in state.active_downloads:
|
||||
try:
|
||||
handle = state.active_downloads[magnet_uri]
|
||||
status = handle.status()
|
||||
|
||||
if status.has_metadata:
|
||||
filepath = os.path.join(status.save_path, status.name)
|
||||
|
||||
if not os.path.exists(filepath):
|
||||
|
||||
consoleLog(f"File Deleted, redownloading: {status.name}")
|
||||
state.dl_session.remove_torrent(handle)
|
||||
del state.active_downloads[magnet_uri]
|
||||
else:
|
||||
consoleLog("Skipping Downloading, download already running...")
|
||||
return False
|
||||
else:
|
||||
consoleLog("Skipping Downloading, download already running... ")
|
||||
return False
|
||||
except RuntimeError as e:
|
||||
consoleLog(f"Error in LibTorrent Handle: {e}")
|
||||
del state.active_downloads[magnet_uri]
|
||||
|
||||
try:
|
||||
params = lt.parse_magnet_uri(magnet_uri)
|
||||
params.save_path = state.download_path
|
||||
|
||||
handle = state.dl_session.add_torrent(params)
|
||||
|
||||
while not handle.has_metadata():
|
||||
time.sleep(1)
|
||||
|
||||
total_size = handle.get_torrent_info().total_size()
|
||||
|
||||
if free_space > total_size:
|
||||
magnetdl = lt.parse_magnet_uri(magnet_uri)
|
||||
magnetdl.save_path = state.download_path
|
||||
download = state.dl_session.add_torrent(magnetdl)
|
||||
else:
|
||||
state.dl_session.remove_torrent(handle)
|
||||
consoleLog("Not enough free space to download this item.")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
consoleLog(f"Failed to add torrent or fetch info: {e}")
|
||||
return False
|
||||
if download:
|
||||
state.active_downloads[magnet_uri] = download
|
||||
consoleLog(f"Added {magnet_uri} to downloads")
|
||||
|
||||
run_thread(threading.Thread(target=dl_status_loop))
|
||||
return True
|
||||
|
||||
|
||||
def add_seed(magnet_uri, file_path):
|
||||
if state.active_downloads is None:
|
||||
state.active_downloads = {}
|
||||
|
||||
init_session()
|
||||
|
||||
if magnet_uri in state.active_downloads:
|
||||
consoleLog("Already seeding this torrent")
|
||||
return False
|
||||
|
||||
try:
|
||||
magnetdl = lt.parse_magnet_uri(magnet_uri)
|
||||
magnetdl.save_path = os.path.dirname(file_path)
|
||||
handle = state.dl_session.add_torrent(magnetdl)
|
||||
except Exception as e:
|
||||
consoleLog(f"Failed to add seed: {e}")
|
||||
return False
|
||||
state.active_downloads[magnet_uri] = handle
|
||||
state.seeded_magnets.add(magnet_uri)
|
||||
return True
|
||||
|
||||
|
||||
def dl_status_loop():
|
||||
global loop_running
|
||||
if loop_running == True:
|
||||
return
|
||||
|
||||
loop_running = True
|
||||
completed_set = set()
|
||||
|
||||
if not state.active_downloads:
|
||||
consoleLog("No active downloads")
|
||||
loop_running = False
|
||||
return
|
||||
|
||||
while state.active_downloads and not state.shutdown_event.is_set():
|
||||
for magnet_uri, magnetdl in list(state.active_downloads.items()):
|
||||
try:
|
||||
status = magnetdl.status()
|
||||
except RuntimeError:
|
||||
continue
|
||||
|
||||
if status.state == lt.torrent_status.seeding and magnet_uri not in completed_set:
|
||||
consoleLog(f"Download completed: {status.name}")
|
||||
|
||||
completed_set.add(magnet_uri)
|
||||
|
||||
if not state.active_downloads:
|
||||
loop_running = False
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
loop_running = False
|
||||
|
||||
def update_settings():
|
||||
|
||||
if state.dl_session is None:
|
||||
return
|
||||
|
||||
settings = {
|
||||
"upload_rate_limit": state.up_speed_limit * 1024,
|
||||
"download_rate_limit": state.down_speed_limit * 1024,
|
||||
"connections_limit": state.max_connections,
|
||||
"active_downloads": state.max_downloads
|
||||
}
|
||||
|
||||
if state.bound_interface:
|
||||
interface_ip = get_interface_ip(state.bound_interface)
|
||||
if interface_ip:
|
||||
settings["outgoing_interfaces"] = interface_ip
|
||||
settings["listen_interfaces"] = f"{interface_ip}:6881"
|
||||
consoleLog(f"Binding to Interface IP: {interface_ip}")
|
||||
else:
|
||||
consoleLog(f"Skipping Binding, no Interface set. ({state.bound_interface})")
|
||||
|
||||
state.dl_session.apply_settings(settings)
|
||||
|
||||
def update_bound_interface():
|
||||
|
||||
settings = { "outgoing_interfaces": state.bound_interface }
|
||||
|
||||
state.dl_session.apply_settings(settings)
|
||||
@@ -0,0 +1,84 @@
|
||||
from utils.logging.logs import update_download_completed_by_hash
|
||||
from utils.logging.logs import consoleLog
|
||||
from utils.data.state import state
|
||||
from plyer import notification
|
||||
import libtorrent as lt
|
||||
import time
|
||||
import os
|
||||
|
||||
|
||||
def cleanup_session():
|
||||
if state.dl_session is not None:
|
||||
for magnetdl in state.active_downloads.values():
|
||||
if hasattr(magnetdl, 'pause'): # Check it's a handle
|
||||
magnetdl.pause()
|
||||
|
||||
del state.dl_session
|
||||
state.dl_session = None
|
||||
state.active_downloads.clear()
|
||||
|
||||
|
||||
|
||||
|
||||
def send_notification(shutdown_event):
|
||||
notified = set()
|
||||
while not shutdown_event.is_set():
|
||||
try:
|
||||
for magnet_uri, magnetdl in list(state.active_downloads.items()):
|
||||
if isinstance(magnetdl, dict):
|
||||
continue
|
||||
|
||||
status = magnetdl.status()
|
||||
|
||||
if status.state == lt.torrent_status.seeding and magnet_uri not in notified:
|
||||
notification.notify(
|
||||
title="Download finished",
|
||||
message=f"{status.name} has finished downloading.",
|
||||
timeout=4
|
||||
)
|
||||
notified.add(magnet_uri)
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while sending notification: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def update_log(shutdown_event):
|
||||
updated = set()
|
||||
while not shutdown_event.is_set():
|
||||
try:
|
||||
for magnet_uri, magnetdl in list(state.active_downloads.items()):
|
||||
if isinstance(magnetdl, dict):
|
||||
continue
|
||||
|
||||
status = magnetdl.status()
|
||||
|
||||
if status.state == lt.torrent_status.seeding and magnet_uri not in updated and magnet_uri not in state.seeded_magnets:
|
||||
consoleLog(f"Marking {status.name} as completed")
|
||||
if hasattr(status, 'info_hashes'):
|
||||
info_hash = str(status.info_hashes.v1)
|
||||
else:
|
||||
info_hash = str(status.info_hash)
|
||||
update_download_completed_by_hash(info_hash, True)
|
||||
updated.add(magnet_uri)
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while updating log file: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def check_deleted_files(shutdown_event):
|
||||
while not shutdown_event.is_set():
|
||||
try:
|
||||
for magnet_uri, magnetdl in list(state.active_downloads.items()):
|
||||
if isinstance(magnetdl, dict):
|
||||
continue
|
||||
|
||||
status = magnetdl.status()
|
||||
|
||||
if status.state == lt.torrent_status.seeding and status.has_metadata:
|
||||
file_path = os.path.join(status.save_path, status.name)
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
consoleLog(f"Registered File Deletion: {status.name}")
|
||||
state.dl_session.remove_torrent(magnetdl)
|
||||
del state.active_downloads[magnet_uri]
|
||||
except Exception as e:
|
||||
consoleLog(f"Exception while checking for file deletions: {e}")
|
||||
time.sleep(5)
|
||||
@@ -0,0 +1,10 @@
|
||||
from network.libtorrent_int import add_download
|
||||
from utils.logging.logs import consoleLog
|
||||
|
||||
|
||||
def add_magnet(uri):
|
||||
if uri is not None and uri.startswith("magnet:?"):
|
||||
add_download(uri)
|
||||
consoleLog("Magnet URI added to LibTorrent")
|
||||
else:
|
||||
consoleLog(f"Invalid Magnet Link: {uri}")
|
||||
Reference in New Issue
Block a user