mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-04 09:59:41 +02:00
refactor: rename scraping functions for consistency and improve data handling
This commit is contained in:
@@ -57,7 +57,7 @@ def _get_telegram_posts():
|
||||
|
||||
return(posts)
|
||||
|
||||
def scrape_monkrus_telegram(query):
|
||||
def scrape_m0nkrus(query):
|
||||
posts = _get_telegram_posts()
|
||||
filtered_posts = []
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@ from core.utils.data.state import state
|
||||
from core.utils.logging.logs import consoleLog
|
||||
|
||||
|
||||
def scrape_rutracker(search_text):
|
||||
search = requests.get(f"{state.api_url}/search?q={search_text}")
|
||||
def scrape_rutracker(query):
|
||||
search = requests.get(f"{state.api_url}/search?q={query}")
|
||||
consoleLog("Sent request to server")
|
||||
if search:
|
||||
try:
|
||||
|
||||
@@ -1,83 +1,43 @@
|
||||
import requests
|
||||
import threading
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import urljoin
|
||||
from core.utils.data.state import state
|
||||
from core.utils.logging.logs import consoleLog
|
||||
from core.utils.general.wrappers import run_thread
|
||||
|
||||
def scrape_uztracker(query):
|
||||
base_url="https://uztracker.net/"
|
||||
search_url = f"{base_url.rstrip('/')}/tracker.php?nm={query}"
|
||||
consoleLog(search_url)
|
||||
posts = []
|
||||
|
||||
global url_uztracker
|
||||
url_uztracker = "https://uztracker.net/tracker.php?nm="
|
||||
|
||||
def init_uztracker():
|
||||
global up
|
||||
global soup
|
||||
try:
|
||||
response = requests.get(url_uztracker, timeout=10)
|
||||
if response.status_code == 200:
|
||||
up = True
|
||||
else:
|
||||
up = False
|
||||
consoleLog(f"Uztracker seems down, status code: {response.status_code}")
|
||||
except requests.exceptions.RequestException as e:
|
||||
consoleLog(f"Request Exception on {url_uztracker}:")
|
||||
consoleLog(str(e))
|
||||
consoleLog("Is the Site down?")
|
||||
up = False
|
||||
|
||||
run_thread(threading.Thread(target=init_uztracker))
|
||||
|
||||
def scrape_uztracker(search):
|
||||
if up:
|
||||
search_url = url_uztracker + search
|
||||
consoleLog(search_url)
|
||||
result = False
|
||||
global results
|
||||
global resulttitles
|
||||
results = []
|
||||
resulttitles = []
|
||||
|
||||
try:
|
||||
resultCount = 0
|
||||
response = requests.get(search_url)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
|
||||
for link in links:
|
||||
if link.b:
|
||||
resultCount += 1
|
||||
results.append(link['href'])
|
||||
resulttitles.append(link.b.text)
|
||||
result = True
|
||||
|
||||
if result:
|
||||
return resulttitles, results
|
||||
|
||||
if not result:
|
||||
# add popup in gui for no result
|
||||
return None
|
||||
|
||||
except requests.RequestException as e:
|
||||
if result:
|
||||
consoleLog(f"Failed to fetch {search_url}: {e}")
|
||||
return None
|
||||
else:
|
||||
consoleLog("Error: Uztracker down")
|
||||
return None, None
|
||||
|
||||
def get_post_title(post_url):
|
||||
if up:
|
||||
response = requests.get(post_url)
|
||||
response = requests.get(search_url)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
maintitle = soup.find(class_='tt-text')
|
||||
if maintitle:
|
||||
return maintitle.text
|
||||
else:
|
||||
consoleLog("Program not found")
|
||||
else:
|
||||
consoleLog("Error: Uztracker down")
|
||||
return None
|
||||
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
|
||||
|
||||
for link in links:
|
||||
if link.b:
|
||||
title = link.b.text
|
||||
url = urljoin(base_url, link['href'])
|
||||
|
||||
author_link = link.find_parent('td').find('a', class_="med ts-text")
|
||||
author = author_link.text.strip() if author_link else "Unknown"
|
||||
|
||||
posts.append(dict(
|
||||
title=title,
|
||||
url=url,
|
||||
author=author
|
||||
))
|
||||
|
||||
return posts
|
||||
|
||||
except requests.RequestException as e:
|
||||
if posts:
|
||||
consoleLog(f"Failed to fetch {search_url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user