Merge branch 'feat-steamrip-noschxl' into merge_candidate_noschxl, improve stuff (http dl impl still needed)

src/core/data/scrapers/monkrus.py

@@ -1,4 +1,8 @@
+from core.utils.data.state import state
+from core.utils.network.jsonhandler import format_data
+from core.utils.data.tracker import get_magnet_link
 from bs4 import BeautifulSoup
+from typing import Dict
 import requests
 import time
 
@@ -31,7 +35,7 @@ def _get_telegram_posts():
         
         for bubble in bubbles:
             post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
-            if not post_txt or not post_txt.b:
+            if post_txt is None or post_txt.b is None:
                 continue
             
             title = post_txt.b.text
@@ -42,9 +46,9 @@ def _get_telegram_posts():
                 if post_url not in added:
                     added.add(post_url)
                     posts.append(dict(
+                        title=title,
                         author="m0nkrus",
                         id=len(posts) + 1,
-                        title=title,
                         url=post_url
                     ))
         
@@ -67,4 +71,18 @@ def scrape_m0nkrus(query):
             filtered_post["id"] = len(filtered_posts) + 1
             filtered_posts.append(filtered_post)
 
-    return filtered_posts
+    return filtered_posts
+
+def get_magnet(post: Dict):
+    return get_magnet_link(post["url"])
+
+Metadata = {
+    "name" : "m0nkrus",
+    "headers" : ["Post Title", "Author"],
+    "scrapeFunc" : scrape_m0nkrus,
+    "linkFunc" : get_magnet,
+    "isMagnet" : True,
+}
+
+def init_m0nkrus():
+    state.trackers.update({Metadata["name"] : Metadata})

src/core/data/scrapers/provider/buzzheavier.py

@@ -0,0 +1,18 @@
+import requests
+
+def scrape_buzzheavier(url):
+
+    response = requests.get(url)
+    response.raise_for_status()
+
+    download_url = url + '/download'
+
+    headers = {
+        'hx-current-url': url,
+        'hx-request': 'true',
+        'referer': url
+    }
+    
+    head_response = requests.head(download_url, headers=headers, allow_redirects=False)
+    hx_redirect = head_response.headers.get('hx-redirect')
+    return hx_redirect

src/core/data/scrapers/provider/gofile.py

@@ -0,0 +1,31 @@
+from core.utils.logging.loghandler import consoleLog
+import requests
+import re
+
+
+def scrape_gofile(url):
+
+    filetoken = re.findall(r"(?<=https...gofile.io\/d\/).*", url)[0]
+    session = requests.Session()
+
+    acc = session.post("https://api.gofile.io/accounts")
+    token = acc.json()["data"]["token"]
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "X-Website-Token": "4fd6sg89d7s6", # Maybe make this dynamic in the future
+    }
+
+    r = session.get(
+        f"https://api.gofile.io/contents/{filetoken}",
+        headers=headers,
+    )
+    try:
+
+        temp: dict = r.json()["data"]["children"]
+        child = [key for key in temp.keys()]
+
+        return temp[child[0]]["link"]
+    except:
+        consoleLog("gofile didnt auth you, either the api has changed\n or you sent to many requests, please try again later.\n If it still doesnt work please open an issue on Github.")
+

src/core/data/scrapers/rutracker.py

@@ -1,19 +1,52 @@
 import requests
 from core.utils.data.state import state
 from core.utils.logging.logs import consoleLog
+from core.utils.data.tracker import get_magnet_link
+from core.utils.network.jsonhandler import split_data, format_data
+from typing import Dict
 
 def scrape_rutracker(query):
-    search = requests.get(f"{state.api_url}/search?q={query}")
+    search = requests.get(f"{state.api_url}/search?q={query}", timeout=15)
     consoleLog("Sent request to server")
     if search:
-        try:
-            return search.text
-        except Exception:
-            consoleLog("No results found / No response from server")
-            return None
+        _, data, _, success, cached = split_data(search.text)
+        if cached:
+            consoleLog("Server response cached")
+        if success:
+            sorted_data = []
+
+            for entry in data:
+                sorted_data.append(
+                    {
+                        "title" : entry["title"],
+                        "author" : entry["author"],
+                        "seeders" : entry["seeders"],
+                        "leechers" : entry["leechers"],
+                        "url" : entry["url"],
+                        "id" : entry["id"],
+                    }
+                )
+
+            return sorted_data
+        else: 
+            consoleLog("Scraping failed server-side, unable to fetch posts from rutracker")
+            return []
     else:
-        return None
-        
+        consoleLog("No response from server, returning nothing")
+        return []
 
 
-# This function utilizes the SoftwareManager server - source code can be found under the SoftwareManager-Server repository.
+def get_magnet(post: Dict):
+    _, post_links, _, _, _ = format_data([post])
+    return get_magnet_link(post_links[0])
+
+Metadata = {
+    "name" : "rutracker",
+    "headers" : ["Post Title", "Author", "Seeders", "Leechers"],
+    "scrapeFunc" : scrape_rutracker,
+    "linkFunc" : get_magnet,
+    "isMagnet" : True,
+}
+
+def init_rutracker():
+    state.trackers.update({Metadata["name"] : Metadata})

src/core/data/scrapers/steamrip.py

@@ -0,0 +1,123 @@
+from core.data.scrapers.provider.buzzheavier import scrape_buzzheavier
+from core.data.scrapers.provider.gofile import scrape_gofile
+from core.utils.logging.logs import consoleLog
+from core.utils.data.state import state
+from bs4 import BeautifulSoup
+from typing import Dict
+import webbrowser
+import requests
+import re
+import time
+
+cache = {
+    "data": [],
+    "last_fetched": 0
+}
+
+cache_expiry = 300
+
+def get_Metadata():
+    return Metadata
+
+def scrape_steamrip_links():
+    
+    current_time = time.time()
+
+    if cache["data"] != [] and (current_time - cache["last_fetched"] < cache_expiry):
+        return cache["data"]
+
+    url = f"https://steamrip.com/games-list-page/"
+    response = requests.get(url)
+    text = response.text
+
+    soup = BeautifulSoup(text, "html.parser")
+    games = soup.find_all("li", class_="az-list-item")
+    
+    if len(games) == 0:
+        return []
+
+    links = []
+    names = []
+    for gamehtml in games:
+        link = gamehtml.find("a", href=lambda x: x and x.startswith("/"))
+        links += re.findall(r'(?<=href=")[^"]*', link.__str__())
+        name = gamehtml.find("a", href=lambda x: x and x.startswith("/"))
+        names += re.findall(r'(?<=\/">)[^<]*', name.__str__())
+
+
+
+    # construct the list[dict[str,str]]
+
+    ret = []
+
+    for i in range(len(names)):
+        ret.append({"title" : names[i], "url" : links[i]})
+
+    return ret
+
+def scrape_steamrip_game_downloads(gamelink):
+    url = "https://steamrip.com" + gamelink
+    response = requests.get(url)
+    soup = BeautifulSoup(response.text, "html.parser")
+    download_link_elements = soup.find_all("a",class_="shortc-button")
+    download_links = ["buzzheavier", "gofile"]
+
+    for download_link in download_link_elements:
+        pure = download_link.attrs.get("href")
+        if pure[2] == "b":
+            download_links[0] = "https:" + pure
+        if pure[2] == "g":
+            download_links[1] = "https:" + pure
+        if pure[2] == "v":
+            download_links[2] = "https:" + pure
+        if pure[2] == "m":
+            download_links[3] = "https:" + pure
+
+    ret = []
+
+    for link in download_links:
+        if len(link) != 1:
+            ret.append(link)
+
+    cache["data"] = ret
+
+    return ret
+
+def get_download_link(post: Dict):
+    url = post["url"]
+    links = scrape_steamrip_game_downloads(url)
+    best = ""
+    for link in links:
+        if link[0] == "h":
+            best = link
+            break
+
+    if links.index(best) == 0:
+        return scrape_buzzheavier(best)
+    elif links.index(best) == 1:
+        return scrape_gofile(best)
+    else:
+        consoleLog("Unable to retrieve download link due to captcha, launching browser...")
+        webbrowser.open(best)
+        return None
+
+def filter_steamrip(query: str):
+    games = scrape_steamrip_links()
+    
+    filtered_games = [
+        game for game in games 
+        if query.lower() in game["title"].lower()
+    ]
+    
+    return filtered_games
+
+Metadata = {
+    "name" : "steamrip",
+    "headers" : ["Game"],
+    "scrapeFunc" : filter_steamrip,
+    "linkFunc" : get_download_link,
+    "isMagnet" : False,
+}
+
+def init_steamrip():
+    state.trackers.update({Metadata["name"] : Metadata})

src/core/data/scrapers/uztracker.py

@@ -2,6 +2,10 @@ import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin
 from core.utils.logging.logs import consoleLog
+from core.utils.data.state import state
+from core.utils.data.tracker import get_magnet_link
+from typing import Dict
+
 
 def scrape_uztracker(query):
     base_url="https://uztracker.net/"
@@ -26,12 +30,26 @@ def scrape_uztracker(query):
 
             posts.append(dict(
                 title=title,
+                author=author,
                 url=url,
-                author=author
             ))
 
         return posts
 
     except requests.RequestException as e:
         consoleLog(f"Failed to fetch {search_url}: {e}")
-        return None
+        return None
+
+def get_magnet(post: Dict):
+    return get_magnet_link(post["url"])
+
+Metadata = {
+    "name" : "uztracker",
+    "headers" : ["Post Title", "Author"],
+    "scrapeFunc" : scrape_uztracker,
+    "linkFunc" : get_magnet,
+    "isMagnet" : True,
+}
+
+def init_uztracker():
+    state.trackers.update({Metadata["name"] : Metadata})