diff --git a/src/core/data/scrapers/monkrus.py b/src/core/data/scrapers/monkrus.py index 9d1b812..7acac5e 100644 --- a/src/core/data/scrapers/monkrus.py +++ b/src/core/data/scrapers/monkrus.py @@ -1,16 +1,28 @@ from bs4 import BeautifulSoup import requests -from core.utils.general.logs import consoleLog +import time -def scrape_monkrus_telegram(query): +cache = { + "data": [], + "last_fetched": 0 +} + +cache_expiry = 300 + +def _get_telegram_posts(): post = 100 max_posts = 250 - posts: list[dict] = [] + posts = [] added = set() - + + current_time = time.time() + + if cache["data"] and (current_time - cache["last_fetched"] < cache_expiry): + return cache["data"] + while post <= max_posts: url = f"https://t.me/s/real_monkrus/{post}" - response = requests.get(url) + response = requests.get(url, timeout=10) soup = BeautifulSoup(response.text, "html.parser") bubbles = soup.find_all("div", class_="tgme_widget_message_bubble") @@ -25,7 +37,7 @@ def scrape_monkrus_telegram(query): title = post_txt.b.text link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net")) - if query.lower() in title.lower() and link: + if link: post_url = link["href"] if post_url not in added: added.add(post_url) @@ -39,4 +51,20 @@ def scrape_monkrus_telegram(query): post += 22 posts.reverse() + + cache["data"] = posts + cache["last_fetched"] = current_time + return(posts) + +def scrape_monkrus_telegram(query): + posts = _get_telegram_posts() + filtered_posts = [] + + for post in posts: + if query.lower() in post["title"].lower(): + filtered_post = post.copy() + filtered_post["id"] = len(filtered_posts) + 1 + filtered_posts.append(filtered_post) + + return filtered_posts \ No newline at end of file diff --git a/src/core/data/scrapers/rutracker.py b/src/core/data/scrapers/rutracker.py index 6eee669..58aecf7 100644 --- a/src/core/data/scrapers/rutracker.py +++ b/src/core/data/scrapers/rutracker.py @@ -1,6 +1,6 @@ import requests from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog def scrape_rutracker(search_text): diff --git a/src/core/data/scrapers/uztracker.py b/src/core/data/scrapers/uztracker.py index e32adb9..1531f9c 100644 --- a/src/core/data/scrapers/uztracker.py +++ b/src/core/data/scrapers/uztracker.py @@ -2,7 +2,7 @@ import requests import threading from bs4 import BeautifulSoup from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.utils.general.wrappers import run_thread diff --git a/src/core/interface/dialogs/settings.py b/src/core/interface/dialogs/settings.py index 0126f88..bf2f0e8 100644 --- a/src/core/interface/dialogs/settings.py +++ b/src/core/interface/dialogs/settings.py @@ -1,6 +1,6 @@ from core.utils.config.settings import save_settings from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.interface.utils.tabhelper import general_tab from core.interface.utils.tabhelper import paths_tab from core.interface.utils.tabhelper import network_tab diff --git a/src/core/interface/gui.py b/src/core/interface/gui.py index 6f61d1e..5654b84 100644 --- a/src/core/interface/gui.py +++ b/src/core/interface/gui.py @@ -1,5 +1,5 @@ -from PySide6 import QtWidgets, QtCore, QtGui -from PySide6.QtCore import Qt, QTimer, QModelIndex, QAbstractTableModel, Signal, QEvent, QSize +from PySide6 import QtWidgets +from PySide6.QtCore import QPersistentModelIndex, Qt, QTimer, QModelIndex, QAbstractTableModel, Signal, QEvent, QSize from PySide6.QtWidgets import ( QLineEdit, QTableView, @@ -30,7 +30,7 @@ import time import sys import json import base64 -from core.utils.general.logs import consoleLog, remove_download_log, flush_log_buffer +from core.utils.logging.logs import consoleLog, remove_download_log, flush_log_buffer from core.utils.general.wrappers import run_thread from core.utils.data.state import state from core.utils.network.download import download_selected @@ -41,7 +41,6 @@ from core.interface.dialogs.settings import settings_dialog from core.interface.assets.base64_icons import settings_black_base64 from core.interface.assets.base64_icons import settings_white_base64 from core.interface.assets.base64_icons import logo_base64 -from core.network.libtorrent_misc import cleanup_session from core.utils.general.shutdown import closehelper @@ -183,19 +182,19 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): super().__init__() self.headers = ["Action", "Name", "Status", "Progress", "Speed", "Size", "Total Size"] - def rowCount(self, parent=QModelIndex()): + def rowCount(self, parent: QModelIndex | QPersistentModelIndex = QModelIndex()) -> int: return len(state.active_downloads) - def columnCount(self, parent=QModelIndex()): + def columnCount(self, parent: QModelIndex | QPersistentModelIndex = QModelIndex()): return len(self.headers) - def headerData(self, section, orientation, role=Qt.DisplayRole): - if role == Qt.DisplayRole and orientation == Qt.Horizontal: + def headerData(self, section, orientation, role: int = Qt.ItemDataRole.DisplayRole): + if role == Qt.ItemDataRole.DisplayRole and orientation == Qt.Orientation.Horizontal: return self.headers[section] return None - def data(self, index, role=Qt.DisplayRole): - if role == Qt.DisplayRole: + def data(self, index: QModelIndex | QPersistentModelIndex, role: int = Qt.ItemDataRole.DisplayRole): + if role == Qt.ItemDataRole.DisplayRole: col = index.column() if index.row() >= len(state.active_downloads) or index.row() < 0: @@ -211,9 +210,9 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): elif col == 1: return status.name if status.has_metadata else "Fetching metadata..." elif col == 2: - if status.state == lt.torrent_status.downloading: + if status.state == lt.torrent_status.downloading: return "Downloading" - elif status.state == lt.torrent_status.seeding: + elif status.state == lt.torrent_status.seeding: return "Seeding" elif status.paused: return "Paused" @@ -257,7 +256,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): else: return "∞" if status.paused else "Stalled" - if role == Qt.UserRole and index.column() == 0: + if role == Qt.ItemDataRole.UserRole and index.column() == 0: magnet_link = list(state.active_downloads.keys())[index.row()] magnetdl = state.active_downloads[magnet_link] return magnetdl.status().paused diff --git a/src/core/interface/utils/searchhelper.py b/src/core/interface/utils/searchhelper.py index af499b1..b38bc2b 100644 --- a/src/core/interface/utils/searchhelper.py +++ b/src/core/interface/utils/searchhelper.py @@ -1,5 +1,5 @@ from PySide6.QtWidgets import QTableWidgetItem -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.data.scrapers.uztracker import scrape_uztracker from core.data.scrapers.rutracker import scrape_rutracker from core.data.scrapers.monkrus import scrape_monkrus_telegram diff --git a/src/core/network/interface.py b/src/core/network/interface.py index 8845d9b..0307ea2 100644 --- a/src/core/network/interface.py +++ b/src/core/network/interface.py @@ -1,4 +1,4 @@ -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.utils.data.state import state import psutil diff --git a/src/core/network/libtorrent_int.py b/src/core/network/libtorrent_int.py index 89751a2..b5fcb45 100644 --- a/src/core/network/libtorrent_int.py +++ b/src/core/network/libtorrent_int.py @@ -4,7 +4,7 @@ from core.network.interface import get_interface_ip import threading import libtorrent as lt from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog global loop_running diff --git a/src/core/network/libtorrent_misc.py b/src/core/network/libtorrent_misc.py index 3760796..98ec561 100644 --- a/src/core/network/libtorrent_misc.py +++ b/src/core/network/libtorrent_misc.py @@ -1,6 +1,6 @@ from core.utils.data.state import state -from core.utils.general.logs import update_download_completed_by_hash -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import update_download_completed_by_hash +from core.utils.logging.logs import consoleLog from plyer import notification import libtorrent as lt import time @@ -57,4 +57,4 @@ def update_log(shutdown_event): updated.add(magnet_uri) except Exception: pass - time.sleep(5) \ No newline at end of file + time.sleep(5) diff --git a/src/core/network/libtorrent_wrapper.py b/src/core/network/libtorrent_wrapper.py index f383cb9..3720353 100644 --- a/src/core/network/libtorrent_wrapper.py +++ b/src/core/network/libtorrent_wrapper.py @@ -1,4 +1,4 @@ -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.network.libtorrent_int import add_download def add_magnet(uri): diff --git a/src/core/utils/config/settings.py b/src/core/utils/config/settings.py index 5550cd2..289c25e 100644 --- a/src/core/utils/config/settings.py +++ b/src/core/utils/config/settings.py @@ -1,6 +1,6 @@ from core.network.libtorrent_int import update_settings from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from .config import create_config diff --git a/src/core/utils/data/models.py b/src/core/utils/data/models.py index 339da08..2c959c7 100644 --- a/src/core/utils/data/models.py +++ b/src/core/utils/data/models.py @@ -7,6 +7,7 @@ class Download: title: str url: str magnet_uri: str + path: str completed: bool @dataclass diff --git a/src/core/utils/data/state.py b/src/core/utils/data/state.py index 717729f..0d38ce8 100644 --- a/src/core/utils/data/state.py +++ b/src/core/utils/data/state.py @@ -9,8 +9,8 @@ class AppState(QObject): def __init__(self): super().__init__() self.posts: List[Dict[str, Any]] = [] - self.post_titles: List[str] = [] - self.post_urls: List[str] = [] + self.post_titles: Optional[List] = None + self.post_urls: Optional[List] = None self.post_author: List[str] = [] self.post_seeders: List[str] = [] self.post_leechers: List[str] = [] @@ -29,11 +29,11 @@ class AppState(QObject): self.max_downloads: int = 10 self.settings_path: str = "" self.dl_session: Any = None - self.active_downloads: List[str] = {} + self.active_downloads: Dict = {} self.window_transparency: bool = False self.interfaces: List = [] self.active_interfaces: List = [] - self.bound_interface: str = None + self.bound_interface: Any = None @property def image_path(self) -> str: diff --git a/src/core/utils/data/tracker.py b/src/core/utils/data/tracker.py index 02d8e2f..962fa38 100644 --- a/src/core/utils/data/tracker.py +++ b/src/core/utils/data/tracker.py @@ -1,7 +1,7 @@ import requests from bs4 import BeautifulSoup from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.utils.network.jsonhandler import format_data diff --git a/src/core/utils/general/loghandler.py b/src/core/utils/logging/loghandler.py similarity index 91% rename from src/core/utils/general/loghandler.py rename to src/core/utils/logging/loghandler.py index f064e00..9a424a3 100644 --- a/src/core/utils/general/loghandler.py +++ b/src/core/utils/logging/loghandler.py @@ -1,4 +1,4 @@ -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.utils.network.download import run_download_direct def split_data(data): diff --git a/src/core/utils/general/logs.py b/src/core/utils/logging/logs.py similarity index 92% rename from src/core/utils/general/logs.py rename to src/core/utils/logging/logs.py index 49f0ae1..7fd0ca9 100644 --- a/src/core/utils/general/logs.py +++ b/src/core/utils/logging/logs.py @@ -5,6 +5,7 @@ from datetime import datetime import json import os import re +import time _log_buffer = [] @@ -21,7 +22,21 @@ def add_download_log(title, url, magnet_uri, completed) -> DownloadList: downloads = [] else: downloads = [] - + + magnetdl = state.active_downloads.get(magnet_uri) + if magnetdl: + status = magnetdl.status() + timeout = 60 + start = time.time() + while not status.has_metadata and (time.time() - start) < timeout: + time.sleep(0.5) + status = magnetdl.status() + torrent_name = status.name + save_path = status.save_path + consoleLog(f"Torrent name (has_metadata={status.has_metadata}): {torrent_name}") + path = os.path.join(save_path, torrent_name) + else: + path = os.path.join(state.download_path, title) if any(d.magnet_uri == magnet_uri or d.url == url for d in downloads): if magnet_uri in state.active_downloads: @@ -37,7 +52,8 @@ def add_download_log(title, url, magnet_uri, completed) -> DownloadList: title=title, url=url, magnet_uri=magnet_uri, - completed=completed, + path=path, + completed=completed )) consoleLog(f"Added {title} to Log File") diff --git a/src/core/utils/network/download.py b/src/core/utils/network/download.py index 4863f34..31b6bf0 100644 --- a/src/core/utils/network/download.py +++ b/src/core/utils/network/download.py @@ -1,10 +1,11 @@ from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.utils.data.tracker import get_item_url from core.utils.data.tracker import get_magnet_link from core.network.libtorrent_wrapper import add_download from core.utils.general.wrappers import run_thread -from core.utils.general.logs import add_download_log +from core.utils.logging.logs import add_download_log +import time import threading @@ -21,8 +22,8 @@ def run_download(item, posts, post_titles): consoleLog(f"Selected URL: {post_url}") magnet_uri = get_magnet_link(post_url) - add_download_log(item, post_url, magnet_uri, False) add_download(magnet_uri) + add_download_log(item, post_url, magnet_uri, False) def run_download_direct(magnet_uri): add_download(magnet_uri) diff --git a/src/core/utils/network/jsonhandler.py b/src/core/utils/network/jsonhandler.py index 724972e..bd8805f 100644 --- a/src/core/utils/network/jsonhandler.py +++ b/src/core/utils/network/jsonhandler.py @@ -32,4 +32,4 @@ def format_data_m0nkrus(data): -# https://open.spotify.com/track/70vjXI7oBou5buTtgiJeRW?si=25640cc4243c4bf3 +# https://open.spotify.com/track/0F0fMcIdL2FmcC9eYqwazX diff --git a/src/core/utils/network/update_checker.py b/src/core/utils/network/update_checker.py index 708f317..f0acb5d 100644 --- a/src/core/utils/network/update_checker.py +++ b/src/core/utils/network/update_checker.py @@ -1,6 +1,6 @@ import requests from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog def check_for_updates(): url = f"https://api.github.com/repos/KeksPirates/SoftwareManager/releases" diff --git a/src/main.py b/src/main.py index f0f6e48..0f263c0 100644 --- a/src/main.py +++ b/src/main.py @@ -1,13 +1,12 @@ from core.interface.gui import MainWindow from core.utils.data.state import state -from core.utils.general.logs import consoleLog +from core.utils.logging.logs import consoleLog from core.network.libtorrent_misc import send_notification, update_log -from core.utils.general.logs import get_download_logs +from core.utils.logging.logs import get_download_logs from core.utils.general.shutdown import closehelper, shutdown_event from core.utils.general.wrappers import run_thread -from core.utils.general.loghandler import split_data, check_completed +from core.utils.logging.loghandler import split_data, check_completed from core.network.interface import list_interfaces, init_interfaces -from core.network.libtorrent_int import update_bound_interface from core.utils.config.config import read_config from PySide6 import QtWidgets from PySide6.QtCore import Qt @@ -33,7 +32,7 @@ def run_gui(): widget.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) qdarktheme.setup_theme("auto", custom_colors={"background": "#00000000"}) - from core.utils.general.logs import set_main_window + from core.utils.logging.logs import set_main_window set_main_window(widget) widget.show() sys.exit(app.exec())