mirror of
https://github.com/KeksPirates/SoftwareManager.git
synced 2026-08-03 17:39:42 +02:00
fix: fix scraping for uztracker posts to support displaying authors properly
This commit is contained in:
@@ -1,10 +1,7 @@
|
|||||||
import requests
|
import requests
|
||||||
import threading
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
from core.utils.data.state import state
|
|
||||||
from core.utils.logging.logs import consoleLog
|
from core.utils.logging.logs import consoleLog
|
||||||
from core.utils.general.wrappers import run_thread
|
|
||||||
|
|
||||||
def scrape_uztracker(query):
|
def scrape_uztracker(query):
|
||||||
base_url="https://uztracker.net/"
|
base_url="https://uztracker.net/"
|
||||||
@@ -16,27 +13,25 @@ def scrape_uztracker(query):
|
|||||||
response = requests.get(search_url)
|
response = requests.get(search_url)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
soup = BeautifulSoup(response.text, 'html.parser')
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
|
links = soup.find_all('tr', class_="tCenter hl-tr", id=lambda x: x and x.startswith('tor_'))
|
||||||
|
|
||||||
for link in links:
|
for link in links:
|
||||||
if link.b:
|
|
||||||
title = link.b.text
|
theme_link = link.find('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
|
||||||
url = urljoin(base_url, link['href'])
|
url = urljoin(base_url, theme_link['href'])
|
||||||
|
title = theme_link.b.text
|
||||||
|
author_link = link.find('a', class_="med")
|
||||||
|
author = author_link.text.strip() if author_link else "Unknown"
|
||||||
|
|
||||||
author_link = link.find_parent('td').find('a', class_="med ts-text")
|
posts.append(dict(
|
||||||
author = author_link.text.strip() if author_link else "Unknown"
|
title=title,
|
||||||
|
url=url,
|
||||||
posts.append(dict(
|
author=author
|
||||||
title=title,
|
))
|
||||||
url=url,
|
|
||||||
author=author
|
|
||||||
))
|
|
||||||
|
|
||||||
return posts
|
return posts
|
||||||
|
|
||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
if posts:
|
consoleLog(f"Failed to fetch {search_url}: {e}")
|
||||||
consoleLog(f"Failed to fetch {search_url}: {e}")
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -157,7 +157,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
|
|||||||
header = table.horizontalHeader()
|
header = table.horizontalHeader()
|
||||||
header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch)
|
header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch)
|
||||||
header.setSectionResizeMode(1, QHeaderView.ResizeMode.Fixed)
|
header.setSectionResizeMode(1, QHeaderView.ResizeMode.Fixed)
|
||||||
header.resizeSection(1, 200)
|
header.resizeSection(1, 500)
|
||||||
header.setStretchLastSection(False)
|
header.setStretchLastSection(False)
|
||||||
|
|
||||||
return table
|
return table
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def format_data_minimal(data):
|
|||||||
post_titles = [post["title"] for post in data]
|
post_titles = [post["title"] for post in data]
|
||||||
post_links = [post["url"] for post in data]
|
post_links = [post["url"] for post in data]
|
||||||
|
|
||||||
return post_author, post_titles, post_links
|
return post_author, post_titles, post_links
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user