KeksPirates/SoftwareManager
1
1
Fork 0
mirror of https://github.com/KeksPirates/SoftwareManager.git synced 2026-08-04 09:59:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

tried to made uztracker scraper async

Browse Source 
not currently working
This commit is contained in:
KeksNino
2025-10-09 14:17:37 +02:00
parent cdb9257859
commit 662751d7db
2 changed files with 29 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/data/scrapers/uztracker.py
+24 -21
View File
@@ -1,5 +1,6 @@
import requests
import asyncio
import aiohttp
from bs4 import BeautifulSoup
global url_uztracker
@@ -23,7 +24,7 @@ async def init_uztracker():
asyncio.run(init_uztracker())
def scrape_uztracker(search, debug, max_results=450):
async def scrape_uztracker(search, debug, max_results=450):
if up:
result = False
global results
@@ -38,27 +39,29 @@ def scrape_uztracker(search, debug, max_results=450):
if debug:
print(search_url)
try:
resultCount = 0
response = requests.get(search_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links:
if link.b:
resultCount += 1
results.append(link['href'])
resulttitles.append(link.b.text)
result = True
async with aiohttp.ClientSession() as session:
try:
async with session.get(search_url) as response:
response.raise_for_status()
text = await response.text()
soup = BeautifulSoup(text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
resultCount = 0
for link in links:
if link.b:
resultCount += 1
results.append(link['href'])
resulttitles.append(link.b.text)
result = True
if resultCount == 0:
# add popup in gui for no result
break
except requests.RequestException as e:
if result:
print(f"Failed to fetch {search_url}: {e}")
return None
if resultCount == 0:
# add popup in gui for no result
break
except aiohttp.ClientError as e:
if result:
print(f"Failed to fetch {search_url}: {e}")
return None
else:
print("Error: Uztracker down")