refactor: rename scraping functions for consistency and improve data handling

This commit is contained in:
Vxrtrauter
2026-03-04 19:11:31 +01:00
parent 90124fd9c8
commit 34a50e1a1b
5 changed files with 48 additions and 105 deletions
+30 -70
View File
@@ -1,83 +1,43 @@
import requests
import threading
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from core.utils.data.state import state
from core.utils.logging.logs import consoleLog
from core.utils.general.wrappers import run_thread
def scrape_uztracker(query):
base_url="https://uztracker.net/"
search_url = f"{base_url.rstrip('/')}/tracker.php?nm={query}"
consoleLog(search_url)
posts = []
global url_uztracker
url_uztracker = "https://uztracker.net/tracker.php?nm="
def init_uztracker():
global up
global soup
try:
response = requests.get(url_uztracker, timeout=10)
if response.status_code == 200:
up = True
else:
up = False
consoleLog(f"Uztracker seems down, status code: {response.status_code}")
except requests.exceptions.RequestException as e:
consoleLog(f"Request Exception on {url_uztracker}:")
consoleLog(str(e))
consoleLog("Is the Site down?")
up = False
run_thread(threading.Thread(target=init_uztracker))
def scrape_uztracker(search):
if up:
search_url = url_uztracker + search
consoleLog(search_url)
result = False
global results
global resulttitles
results = []
resulttitles = []
try:
resultCount = 0
response = requests.get(search_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links:
if link.b:
resultCount += 1
results.append(link['href'])
resulttitles.append(link.b.text)
result = True
if result:
return resulttitles, results
if not result:
# add popup in gui for no result
return None
except requests.RequestException as e:
if result:
consoleLog(f"Failed to fetch {search_url}: {e}")
return None
else:
consoleLog("Error: Uztracker down")
return None, None
def get_post_title(post_url):
if up:
response = requests.get(post_url)
response = requests.get(search_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
maintitle = soup.find(class_='tt-text')
if maintitle:
return maintitle.text
else:
consoleLog("Program not found")
else:
consoleLog("Error: Uztracker down")
return None
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links:
if link.b:
title = link.b.text
url = urljoin(base_url, link['href'])
author_link = link.find_parent('td').find('a', class_="med ts-text")
author = author_link.text.strip() if author_link else "Unknown"
posts.append(dict(
title=title,
url=url,
author=author
))
return posts
except requests.RequestException as e:
if posts:
consoleLog(f"Failed to fetch {search_url}: {e}")
return None