KeksPirates/SoftwareManager
1
1
Fork 0
mirror of https://github.com/KeksPirates/SoftwareManager.git synced 2026-08-04 09:59:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

add more search results for uztracker

Browse Source
This commit is contained in:
KeksNino
2025-10-06 00:10:54 +02:00
parent 7c9bad2451
commit ca28097dd9
1 changed files with 37 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/scraping/uztracker_scraper.py
+37 -30
View File
@@ -20,44 +20,51 @@ async def init_uztracker():
asyncio.run(init_uztracker()) asyncio.run(init_uztracker())
def scrape_uztracker(search, debug): def scrape_uztracker(search, debug, max_results=450):
if up: if up:
search_url = url_uztracker + search
if debug:
print(search_url)
result = False result = False
global results global results
global resulttitles global resulttitles
results = [] results = []
resulttitles = [] resulttitles = []
try: per_page = 50
resultCount = 0
response = requests.get(search_url) for start in range(0, max_results, per_page):
response.raise_for_status() search_url = url_uztracker + search + f"&start={start}"
soup = BeautifulSoup(response.text, 'html.parser') if debug:
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic')) print(search_url)
for link in links:
if link.b: try:
resultCount += 1 resultCount = 0
results.append(link['href']) response = requests.get(search_url)
resulttitles.append(link.b.text) response.raise_for_status()
result = True soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
if result: for link in links:
return resulttitles, results if link.b:
resultCount += 1
if not result: results.append(link['href'])
# add popup in gui for no result resulttitles.append(link.b.text)
result = True
if resultCount == 0:
# add popup in gui for no result
break
except requests.RequestException as e:
if result:
print(f"Failed to fetch {search_url}: {e}")
return None return None
except requests.RequestException as e: else:
if result: print("Error: Uztracker down")
print(f"Failed to fetch {search_url}: {e}") return None, None
return None
else: if not results:
print("Error: Uztracker down") return None, None
return None, None
return resulttitles, results
def get_post_title(post_url, debug): def get_post_title(post_url, debug):
if up: if up: