KeksPirates/SoftwareManager
1
1
Fork 0
mirror of https://github.com/KeksPirates/SoftwareManager.git synced 2026-08-04 17:59:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refactor uztracker scraper to handle site availability checks and improve error handling

Browse Source
This commit is contained in:
Vxrtrauter
2025-09-05 19:10:46 +02:00
parent 922897ebc5
commit b701e75bdf
1 changed files with 79 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/scraping/uztracker_scraper.py
+75 -55
View File
@@ -2,76 +2,96 @@ import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
global url_uztracker global url_uztracker
global up
url_uztracker = "https://uztracker.net/tracker.php?nm=" # placeholder url for now url_uztracker = "https://uztracker.net/tracker.php?nm=" # placeholder url for now
response = requests.get(url_uztracker) # get da content from da url try:
soup = BeautifulSoup(response.content, 'html.parser') # create bs object response = requests.get(url_uztracker) # get da content from da url
soup = BeautifulSoup(response.content, 'html.parser') # create bs object
up = True
except requests.exceptions.RequestException as e:
print(f"\nRequest Exception on {url_uztracker}:")
print(e)
print("\nIs the Site down?")
up = False
def scrape_uztracker(search, debug): def scrape_uztracker(search, debug):
search_url = url_uztracker + search if up:
if debug: search_url = url_uztracker + search
print(search_url) if debug:
result = False print(search_url)
global results result = False
global resulttitles global results
results = [] global resulttitles
resulttitles = [] results = []
resulttitles = []
try: try:
resultCount = 0 resultCount = 0
response = requests.get(search_url) response = requests.get(search_url)
response.raise_for_status() response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser') soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic')) links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links: for link in links:
if link.b: if link.b:
resultCount += 1 resultCount += 1
# print(f"Result found: ({resultCount}) {link.b.text} | {link['href']}") # print(f"Result found: ({resultCount}) {link.b.text} | {link['href']}")
results.append(link['href']) results.append(link['href'])
resulttitles.append(link.b.text) resulttitles.append(link.b.text)
result = True result = True
if result: if result:
return resulttitles, results return resulttitles, results
if not result: if not result:
if debug: if debug:
print(f'No Results found for "{search}"') print(f'No Results found for "{search}"')
# add popup in gui for no result # add popup in gui for no result
return return
except requests.RequestException as e: except requests.RequestException as e:
if result: if result:
print(f"Failed to fetch {search_url}: {e}") print(f"Failed to fetch {search_url}: {e}")
return None return None
else:
print("Error: Uztracker down")
return None, None
def get_post_title(post_url, debug): def get_post_title(post_url, debug):
response = requests.get(post_url) if up:
soup = BeautifulSoup(response.text, 'html.parser') response = requests.get(post_url)
maintitle = soup.find(class_='tt-text') soup = BeautifulSoup(response.text, 'html.parser')
if maintitle: maintitle = soup.find(class_='tt-text')
return maintitle.text if maintitle:
return maintitle.text
else:
if debug:
print("Program not found")
else: else:
if debug: print("Error: Uztracker down")
print("Program not found") return None
def get_magnet_link(post_url, debug): def get_magnet_link(post_url, debug):
try: if up:
response = requests.get(post_url) try:
response.raise_for_status() response = requests.get(post_url)
soup = BeautifulSoup(response.text, 'html.parser') response.raise_for_status()
magnet_link = soup.find('a', href=lambda x: x and x.startswith('magnet:')) soup = BeautifulSoup(response.text, 'html.parser')
if magnet_link: magnet_link = soup.find('a', href=lambda x: x and x.startswith('magnet:'))
if magnet_link:
if debug:
print("Magnet Link Retrieved: ", magnet_link['href'])
return magnet_link['href']
else:
if debug:
print("Magnet Link not Found!")
except requests.RequestException as e:
if debug: if debug:
print("Magnet Link Retrieved: ", magnet_link['href']) print(f"Failed to fetch {post_url}: {e}")
return magnet_link['href'] else:
else: print("Error: Uztracker down")
if debug: return None
print("Magnet Link not Found!")
except requests.RequestException as e:
if debug:
print(f"Failed to fetch {post_url}: {e}")
# What x and x.startswith('magnet:') Does # What x and x.startswith('magnet:') Does
# x - check if x exists (not None/empty) # x - check if x exists (not None/empty)