Merge pull request #79 from KeksPirates/fix/steamrip

Refactor GoFile scraper and add VikingFile and MegaDB support

src/data/hosts/gofile.py

@@ -1,41 +1,64 @@
 from utils.logging.logs import consoleLog
 import requests
+import hashlib
+import time
 import re
 
+_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+_WT_SECRET = "5d4f7g8sd45fsd"
+_API_BASE = "https://api.gofile.io"
+
+
+def _generate_website_token(account_token):
+    time_bucket = str(int(time.time() / 14400))
+    raw = f"{_USER_AGENT}::en-US::{account_token}::{time_bucket}::{_WT_SECRET}"
+    return hashlib.sha256(raw.encode()).hexdigest()
+
 
 def scrape_gofile(url):
-
-    filetoken = re.findall(r"(?<=https...gofile.io\/d\/).*", url)[0]
-    session = requests.Session()
-
-    headers = {
-        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:148.0) Gecko/20100101 Firefox/148.0",
-        "Accept": "*/*",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Accept-Encoding": "gzip, deflate, br, zstd",
-        "Referer": "https://gofile.io/",
-        "Authorization": f"Bearer lUUtRAccOuRUoZhelNSslDFSlpOgKLDj",
-        "X-Website-Token": "ffd9ffa831c50b9e68ca5e65cbbbd1a50e9a32e63022e17c7c6748388a1ed73b",
-        "X-BL": "en-US",
-        "Origin": "https://gofile.io",
-        "Sec-GPC": "1",
-        "Connection": "keep-alive",
-        "TE": "trailers"
-    }
-
-    r = session.get(
-        f"https://api.gofile.io/contents/{filetoken}",
-        headers=headers,
-    )
-
-    try:
-
-        temp: dict = r.json()["data"]["children"]
-        child = [key for key in temp.keys()]
-
-        link = temp[child[0]]["link"]
-        return link, headers
-    except Exception:
-        consoleLog("GoFile Authorization failed, if the Issue persists, please open an Issue on GitHub")
+    match = re.search(r"gofile\.io/d/([a-zA-Z0-9]+)", url)
+    if not match:
+        consoleLog("GoFile: Invalid URL")
         return None
 
+    content_id = match.group(1)
+    session = requests.Session()
+
+    # Create a guest account
+    r = session.post(f"{_API_BASE}/accounts", headers={"User-Agent": _USER_AGENT})
+    data = r.json()
+    if data.get("status") != "ok":
+        consoleLog("GoFile: Failed to create guest account")
+        return None
+
+    account_token = data["data"]["token"]
+    website_token = _generate_website_token(account_token)
+
+    headers = {
+        "User-Agent": _USER_AGENT,
+        "Authorization": f"Bearer {account_token}",
+        "X-Website-Token": website_token,
+        "X-BL": "en-US",
+        "Referer": "https://gofile.io/",
+        "Origin": "https://gofile.io",
+    }
+
+    # Fetch folder contents
+    r = session.get(f"{_API_BASE}/contents/{content_id}", headers=headers)
+    data = r.json()
+    if data.get("status") != "ok":
+        consoleLog(f"GoFile: API error - {data.get('status')}")
+        return None
+
+    children = data["data"].get("children", {})
+    if not children:
+        consoleLog("GoFile: No files found")
+        return None
+
+    first_child = next(iter(children.values()))
+    link = first_child.get("link")
+    if not link:
+        consoleLog("GoFile: No download link in response")
+        return None
+
+    return link, headers

src/data/hosts/megadb.py

@@ -0,0 +1,14 @@
+from utils.logging.logs import consoleLog
+import webbrowser
+import re
+
+
+def scrape_megadb(url):
+    match = re.search(r"megadb\.net/([a-zA-Z0-9]+)", url)
+    if not match:
+        consoleLog("MegaDB: Invalid URL")
+        return None
+
+    consoleLog("MegaDB: Captcha required, launching browser...")
+    webbrowser.open(url)
+    return None

src/data/hosts/vikingfile.py

@@ -0,0 +1,14 @@
+from utils.logging.logs import consoleLog
+import webbrowser
+import re
+
+
+def scrape_vikingfile(url):
+    match = re.search(r"vikingfile\.com/f/([a-zA-Z0-9]+)", url)
+    if not match:
+        consoleLog("VikingFile: Invalid URL")
+        return None
+
+    consoleLog("VikingFile: Captcha required, launching browser...")
+    webbrowser.open(url)
+    return None

src/data/sources/steamrip.py

@@ -1,5 +1,7 @@
 from data.hosts.buzzheavier import scrape_buzzheavier
 from data.hosts.gofile import scrape_gofile
+from data.hosts.vikingfile import scrape_vikingfile
+from data.hosts.megadb import scrape_megadb
 from utils.logging.logs import consoleLog
 from bs4 import BeautifulSoup
 from typing import Dict
@@ -25,9 +27,14 @@ class SteamripScraper:
         if self.cache["data"] != [] and (current_time - self.cache["last_fetched"] < self.cache_expiry):
             return self.cache["data"]
 
+        try:
             url = f"https://steamrip.com/games-list-page/"
             response = requests.get(url)
+            response.raise_for_status()
             text = response.text
+        except requests.RequestException as e:
+            consoleLog(f"SteamRip: Failed to fetch games list - {e}")
+            return []
 
         soup = BeautifulSoup(text, "html.parser")
         games = soup.find_all("li", class_="az-list-item")
@@ -55,17 +62,29 @@ class SteamripScraper:
         return ret
 
     def scrape_steamrip_game_downloads(self, url):
+        try:
             response = requests.get(url)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            consoleLog(f"SteamRip: Failed to fetch game page - {e}")
+            return []
+
         soup = BeautifulSoup(response.text, "html.parser")
         download_link_elements = soup.find_all("a",class_="shortc-button")
-        download_links = ["buzzheavier", "gofile"]
+        download_links = ["buzzheavier", "gofile", "vikingfile", "megadb"]
 
         for download_link in download_link_elements:
             pure = download_link.attrs.get("href")
+            if not pure or len(pure) < 3:
+                continue
             if pure[2] == "b":
                 download_links[0] = "https:" + pure
             if pure[2] == "g":
                 download_links[1] = "https:" + pure
+            if pure[2] == "v":
+                download_links[2] = "https:" + pure
+            if pure[2] == "m":
+                download_links[3] = "https:" + pure
 
         ret = []
 
@@ -78,22 +97,47 @@ class SteamripScraper:
     def get_download_link(self, post: Dict):
         url = post["url"]
         links = self.scrape_steamrip_game_downloads(url)
+
+        if not links:
+            consoleLog("SteamRip: No download links found")
+            return None, None
+
         best = ""
         for link in links:
             if link[0] == "h":
                 best = link
                 break
 
-        if links.index(best) == 0:
+        if not best:
+            consoleLog("SteamRip: No valid download links found")
+            return None, None
+
+        try:
+            idx = links.index(best)
+        except ValueError:
+            consoleLog("SteamRip: Failed to resolve download host")
+            return None, None
+
+        try:
+            if idx == 0:
                 link = scrape_buzzheavier(best)
                 return link
-        elif links.index(best) == 1:
+            elif idx == 1:
                 link, headers = scrape_gofile(best)
                 return link, headers
+            elif idx == 2:
+                scrape_vikingfile(best)
+                return None, None
+            elif idx == 3:
+                scrape_megadb(best)
+                return None, None
             else:
-            consoleLog("Unable to retrieve download link due to captcha, launching browser...")
+                consoleLog("SteamRip: Unknown host, launching browser...")
                 webbrowser.open(best)
                 return None, None
+        except Exception as e:
+            consoleLog(f"SteamRip: Scraper failed - {e}")
+            return None, None
 
     def search(self, query: str):
         games = self.scrape_steamrip_links()