diff --git a/src/core/data/scrapers/monkrus.py b/src/core/data/scrapers/monkrus.py index c7af80f..d41c020 100644 --- a/src/core/data/scrapers/monkrus.py +++ b/src/core/data/scrapers/monkrus.py @@ -1,11 +1,8 @@ from bs4 import BeautifulSoup import requests import time +from core.utils.data.state import state -Metadata = { - "headers" : ["Post Title", "Author"], - "name" : "m0nkrus", -} def get_Metadata(): return Metadata @@ -77,5 +74,14 @@ def scrape_m0nkrus(query): return filtered_posts -if __name__ == "__main__": - print(scrape_m0nkrus("adobe")) + +Metadata = { + "name" : "m0nkrus", + "headers" : ["Post Title", "Author"], + "searchkey" : None, + "scrapeFunc" : scrape_m0nkrus, + "scrapeSearches" : True, +} + +if __name__ != "__main__": + state.trackers.update({Metadata["name"] : Metadata}) diff --git a/src/core/data/scrapers/rutracker.py b/src/core/data/scrapers/rutracker.py index 2266ead..5641cdf 100644 --- a/src/core/data/scrapers/rutracker.py +++ b/src/core/data/scrapers/rutracker.py @@ -1,27 +1,29 @@ import requests from core.utils.data.state import state from core.utils.logging.logs import consoleLog +from core.utils.network.jsonhandler import split_data -Metadata = { - "headers" : ["Post Title", "Author", "Seeders", "Leechers"], - "name" : "rutracker", -} - -def get_Metadata(): - return Metadata def scrape_rutracker(query): search = requests.get(f"{state.api_url}/search?q={query}") - consoleLog("Sent request to server") + consoleLog("[core.data.scrapers.rutracker] Sent request to server") if search: - try: - return search.text - except Exception: - consoleLog(f"No results found / No response from server, exception: {Exception}") - return None + _, data, _, _, _ = split_data(search.text) + + return data else: - return None + consoleLog("[core.data.scrapers.rutracker] No search Text, returning nothing") + return [] +Metadata = { + "name" : "rutracker", + "headers" : ["Post Title", "Author", "Seeders", "Leechers"], + "searchkey" : None, + "scrapeFunc" : scrape_rutracker, + "scrapeSearches" : True, +} -# This function utilizes the SoftwareManager server - source code can be found under the SoftwareManager-Server repository. + +if __name__ != "__main__": + state.trackers.update({Metadata["name"] : Metadata}) diff --git a/src/core/data/scrapers/steamrip.py b/src/core/data/scrapers/steamrip.py index 8d0076e..52aaae1 100644 --- a/src/core/data/scrapers/steamrip.py +++ b/src/core/data/scrapers/steamrip.py @@ -1,21 +1,33 @@ from bs4 import BeautifulSoup +from fuzzyfinder.main import fuzzyfinder import requests import re +import time +from typing import Generator +from core.utils.data.state import state -Metadata = { - "headers" : ["Game", "Availible Downloads"], - "name" : "steamrip", + +cache = { + "data": [], + "last_fetched": 0 } +cache_expiry = 300 + def get_Metadata(): return Metadata -def scrape_steamrip_links(text = None): - if text is None: - url = f"https://steamrip.com/games-list-page/" - response = requests.get(url) - text = response.text +def scrape_steamrip_links(): + + current_time = time.time() + + if cache["data"] != [] and (current_time - cache["last_fetched"] < cache_expiry): + return cache["data"] + + url = f"https://steamrip.com/games-list-page/" + response = requests.get(url) + text = response.text soup = BeautifulSoup(text, "html.parser") games = soup.find_all("li", class_="az-list-item") @@ -50,7 +62,8 @@ def scrape_steamrip_game_downloads(gamelink): download_link_elements = soup.find_all("a",class_="shortc-button") - download_links = ["b", "g", "v", "m"] + + download_links = ["buzzheavier", "gofile", "vikingfile", "megadb"] for download_link in download_link_elements: pure = download_link.attrs.get("href") @@ -69,11 +82,28 @@ def scrape_steamrip_game_downloads(gamelink): if len(link) != 1: ret.append(link) - + cache["data"] = ret return ret +def filter_steamrip(query: str): + + options: Generator = fuzzyfinder(query, scrape_steamrip_links(), accessor=lambda x: x["name"]) + + return list(options) + + + + +Metadata = { + "name" : "steamrip", + "headers" : ["Game", "Available Downloads"], + "scrapeFunc" : filter_steamrip, +} + +if __name__ != "__main__": + state.trackers.update({Metadata["name"] : Metadata}) + if __name__ == "__main__": - #print(offline_scrape_steamrip_links()) print(scrape_steamrip_game_downloads("/r-e-p-o-free-download/")) diff --git a/src/core/data/scrapers/uztracker.py b/src/core/data/scrapers/uztracker.py index a110e00..0746a2d 100644 --- a/src/core/data/scrapers/uztracker.py +++ b/src/core/data/scrapers/uztracker.py @@ -2,11 +2,8 @@ import requests from bs4 import BeautifulSoup from urllib.parse import urljoin from core.utils.logging.logs import consoleLog +from core.utils.data.state import state -Metadata = { - "headers" : ["Post Title", "Author"], - "name" : "uztracker", -} def get_Metadata(): return Metadata @@ -41,3 +38,14 @@ def scrape_uztracker(query): except requests.RequestException as e: consoleLog(f"Failed to fetch {search_url}: {e}") return None + +Metadata = { + "name" : "uztracker", + "headers" : ["Post Title", "Author"], + "searchkey" : None, + "scrapeFunc" : scrape_uztracker, + "scrapeSearches" : True, +} + +if __name__ != "__main__": + state.trackers.update({Metadata["name"] : Metadata}) diff --git a/src/core/interface/gui.py b/src/core/interface/gui.py index c75f6aa..fc9b209 100644 --- a/src/core/interface/gui.py +++ b/src/core/interface/gui.py @@ -147,29 +147,19 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): flush_log_buffer() - # create tracker tables - state.trackertable - table = QTableWidget() - table.setColumnCount(len(headers)) - table.setEditTriggers(QtWidgets.QAbstractItemView.EditTrigger.NoEditTriggers) - table.verticalHeader().setVisible(False) - table.setHorizontalHeaderLabels(headers) + # create initial tracker table + state.trackertable = QTableWidget() + state.trackertable.setEditTriggers(QtWidgets.QAbstractItemView.EditTrigger.NoEditTriggers) + state.trackertable.verticalHeader().setVisible(False) + state.trackertable.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) + state.trackertable.viewport().setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) + + - header = table.horizontalHeader() - header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch) - header.setSectionResizeMode(1, QHeaderView.ResizeMode.Fixed) - header.resizeSection(1, 500) - header.setStretchLastSection(False) - - table.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) - table.viewport().setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) - - state.tracker_list.update({key: table}) - container = QWidget() containerLayout = QVBoxLayout() containerLayout.addWidget(self.searchbar) - containerLayout.addWidget(state.tracker_list[state.tracker]) + containerLayout.addWidget(state.trackertable) class DownloadModel(QAbstractTableModel): def __init__(self): @@ -350,7 +340,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): delegate.clicked.connect(on_pause_resume_clicked) # download button triggers - self.dlbutton.clicked.connect(lambda: run_thread(threading.Thread(target=download_selected, args=(state.tracker_list[state.tracker].currentItem(), state.posts, state.post_titles)))) + #! THIS IS TEMPORARY self.dlbutton.clicked.connect(lambda: run_thread(threading.Thread(target=download_selected, args=(state.trackertable.currentItem(), state.posts, state.posts)))) container.setLayout(containerLayout) self.setCentralWidget(container) @@ -370,10 +360,11 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): self.horizontal_layout = QHBoxLayout() self.horizontal_layout.addWidget(self.emptyResults, stretch=3) - self.tracker_widget = state.tracker_list[state.tracker] + self.tracker_widget = state.trackertable self.horizontal_layout.addWidget(self.tracker_widget) + self.horizontal_layout.addWidget(state.trackertable) - self.tab1 = create_tab("Search", self.searchbar, state.tracker_list[state.tracker], self.tabs, self.dlbutton, self.horizontal_layout) + self.tab1 = create_tab("Search", self.searchbar, state.trackertable, self.tabs, self.dlbutton, self.horizontal_layout) # self.tab2 = create_tab("Library", self.emptyLibrary, self.libraryList, self.tabs, None, None) self.tab3 = create_tab("Downloads", self.emptyDownload, self.downloadList, self.tabs, None, None) @@ -402,7 +393,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): self.tracker_list = QComboBox() - self.tracker_list.addItems(list(state.tracker_list.keys())) + self.tracker_list.addItems(list(state.trackers.keys())) self.tracker_list.setCursor(Qt.CursorShape.PointingHandCursor) self.tracker_list.activated.connect(self.set_tracker) self.corner_layout.addWidget(self.tracker_list) @@ -504,22 +495,14 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): event.accept() def set_tracker(self, _): - old_tracker = state.tracker - state.tracker = self.tracker_list.currentText() - - old_widget = state.tracker_list[old_tracker] - self.horizontal_layout.removeWidget(old_widget) - old_widget.setParent(None) - - tracker_widget = state.tracker_list[state.tracker] - self.horizontal_layout.addWidget(tracker_widget) + state.currenttracker = self.tracker_list.currentText() def show_empty_results(self, show: bool): if show: - state.tracker_list[state.tracker].hide() + state.trackertable.hide() self.emptyResults.show() else: - state.tracker_list[state.tracker].show() + state.trackertable.show() self.emptyResults.hide() def show_empty_downloads(self): @@ -533,8 +516,8 @@ class MainWindow(QtWidgets.QMainWindow, QWidget): # thank you claude def resizeEvent(self, event): super().resizeEvent(event) - table_width = state.tracker_list[state.tracker].viewport().width() - state.tracker_list[state.tracker].setColumnWidth(1, int(table_width * 0.3)) + table_width = state.trackertable.viewport().width() + state.trackertable.setColumnWidth(1, int(table_width * 0.3)) def contextMenuEvent(self, event: QContextMenuEvent): if self.downloadList.underMouse(): diff --git a/src/core/interface/utils/searchhelper.py b/src/core/interface/utils/searchhelper.py index 9787c12..5898bef 100644 --- a/src/core/interface/utils/searchhelper.py +++ b/src/core/interface/utils/searchhelper.py @@ -1,16 +1,10 @@ -from PySide6.QtWidgets import QTableWidgetItem +from PySide6.QtWidgets import QTableWidgetItem, QHeaderView from core.utils.logging.logs import consoleLog from core.data.scrapers.uztracker import scrape_uztracker from core.data.scrapers.rutracker import scrape_rutracker from core.data.scrapers.monkrus import scrape_m0nkrus +from core.data.scrapers.steamrip import filter_steamrip from core.utils.data.state import state -import fuzzyfinder - -scrapers = { - "uztracker": scrape_uztracker, - "rutracker": scrape_rutracker, - "m0nkrus": scrape_m0nkrus -} def return_pressed(self): search_text = self.searchbar.text() @@ -19,63 +13,23 @@ def return_pressed(self): return consoleLog(f"User searched for: {search_text}") - if state.currenttracker == "rutracker": - response = scrape_rutracker(search_text) - if response: - _, state.posts, _, _, cached = split_data(response) - if state.posts == []: - consoleLog(f"No Results for {search_text}") - state.tracker_list[state.tracker].clear() - self.show_empty_results(True) - else: - state.post_titles, _, state.post_author, state.post_seeders, state.post_leechers = format_data(state.posts) - self.show_empty_results(False) - state.tracker_list[state.tracker].clear() - state.tracker_list[state.tracker].setHorizontalHeaderLabels(["Post Title", "Author", "Seeders", "Leechers"]) - state.tracker_list[state.tracker].setRowCount(len(state.post_titles)) - for i, author in enumerate(state.post_author): - state.tracker_list[state.tracker].setItem(i, 1, QTableWidgetItem(author)) - for i, title in enumerate(state.post_titles): - state.tracker_list[state.tracker].setItem(i, 0, QTableWidgetItem(title)) - for i, seeders in enumerate(state.post_seeders): - state.tracker_list[state.tracker].setItem(i, 2, QTableWidgetItem(seeders)) - for i, leechers in enumerate(state.post_leechers): - state.tracker_list[state.tracker].setItem(i, 3, QTableWidgetItem(leechers)) - if state.debug == True: - consoleLog(f"Response Cached: {cached}") - else: - consoleLog(f"No response from rutracker") - state.tracker_list[state.tracker].clear() - self.show_empty_results(True) + tracker = state.trackers[state.currenttracker] - elif state.tracker is not None: - state.posts = scrapers[state.tracker](search_text) - if state.posts == []: - consoleLog(f"No Results for {search_text}") - state.tracker_list[state.tracker].clear() - self.show_empty_results(True) - else: - state.post_author, state.post_titles, state.post_urls = format_data_minimal(state.posts) - self.show_empty_results(False) - state.tracker_list[state.tracker].clear() - state.tracker_list[state.tracker].setHorizontalHeaderLabels(["Post Title", "Author"]) - state.tracker_list[state.tracker].setRowCount(len(state.post_titles)) - for i, author in enumerate(state.post_author): - state.tracker_list[state.tracker].setItem(i, 1, QTableWidgetItem(author)) - for i, title in enumerate(state.post_titles): - state.tracker_list[state.tracker].setItem(i, 0, QTableWidgetItem(title)) + scrapefunc: function = tracker["scrapeFunc"] + + state.posts = scrapefunc(search_text) - elif state.tracker == "steamrip": - games = scrape_steamrip_links() + state.trackertable.clear() - options: Generator = fuzzyfinder.main.fuzzyfinder(search_text, self.opts, accessor=lambda x: x["name"]) - - filtered_opts = [*options] - - if filtered_opts == []: - consoleLog(f"No results for {search_text}") - state.tracker_list[state.tracker].clear() - self.show_empty_results(True) - else: - state + if state.posts == []: + self.show_empty_results(True) + + state.trackertable.setColumnCount(len(tracker["headers"])) + state.trackertable.setHorizontalHeaderLabels(tracker["headers"]) + state.trackertable.setRowCount(len(state.posts)) + for x, rowdata in enumerate(state.posts): + for y, data in enumerate(rowdata.values()): + state.trackertable.setItem(x, y, QTableWidgetItem(data)) + + consoleLog("[searchhelper] updated table") diff --git a/src/core/utils/data/state.py b/src/core/utils/data/state.py index 6077010..6d135c8 100644 --- a/src/core/utils/data/state.py +++ b/src/core/utils/data/state.py @@ -17,14 +17,13 @@ class AppState(QObject): self.currenttracker: str = "rutracker" self.trackertable: QTableWidget - self.trackers: Dict[str,Dict[str,Any]] # each tracker should add itself here, will be listed in order of module initialisation + self.trackers: Dict[str,Dict[str,Any]] = {}# each tracker should add itself here ''' an example: "rutracker" : { "name" : "rutracker", # name of the tracker - "posts" : [post1, post2, ...], # all the posts, will be in self.posts, "headers" : ["author", "title"], # keys shown in the table - "sortkey" : "name" # the key to sort posts by # currently only usefull to steamrip + "scrapeFunc" : function, } ''' self.api_url: str = "https://api.michijackson.xyz" @@ -42,25 +41,6 @@ class AppState(QObject): self.active_interfaces: List = [] self.bound_interface: Any = None - def update_tracker_table(self, newtracker: str): - - if newtracker not in self.trackers.keys(): - raise ValueError("the selected tracker is not initialised.") - - self.trackertable.setColumnCount(self.trackers[self.currenttracker][]) - - for row_index, row_data in enumerate(self.trackers[self.currenttracker].values()): - for col_index, key in enumerate(row_data.keys()): - if key in self.trackers[self.currenttracker]["ignorekeys"]: - continue - value = row_data.get(key, "") - self.trackertable.setItem( - row_index, - col_index, - QtWidgets.QTableWidgetItem(value) - ) - - @property def image_path(self) -> str: return self._image_path