Compare commits

..

15 Commits

Author SHA1 Message Date
KeksNino c8a1e72107 remove test.txt 2025-10-18 22:09:22 +02:00
KeksNino 7c505f1402 this is a test 2025-10-18 16:45:09 +02:00
KeksNino 55e4f86a39 update workflows 2025-10-18 15:59:50 +02:00
KeksNino bd71b0e645 update workflows 2025-10-18 15:55:48 +02:00
KeksNino 2a86e4b774 update workflows 2025-10-18 15:50:16 +02:00
shayaa d17b4bd44a add & impl. m0nkrus scraperM 2025-10-18 02:32:33 +02:00
Vxrtrauter 10c935e852 fix tracker.py to handle m0nkrus scraper for downloads 2025-10-18 02:31:00 +02:00
Vxrtrauter bd8320a26c improve monkrus scraper for more results 2025-10-18 02:25:15 +02:00
Vxrtrauter c99ce339e0 fully impl. monkrus scraper (+ add search logic to scraper) 2025-10-18 01:49:00 +02:00
Vxrtrauter 1822a150d7 finnish monkrus scraper (probably) 2025-10-18 01:33:12 +02:00
Vxrtrauter 7e5e3033e6 add scraping logic 2025-10-18 01:22:11 +02:00
Vxrtrauter 1cab671a13 add more testing stuff 2025-10-18 01:08:36 +02:00
Vxrtrauter 5637e80657 added some more testing stuff 2025-10-17 21:23:17 +02:00
Vxrtrauter 2f78b812ba add besic scraping with message retrieving logic; still featuring full testing & debug outpus 2025-10-17 20:23:44 +02:00
Vxrtrauter 5af9e79ecc Refractor searchhelper to finally be readable, reverd uztracker scraper to old format since i didn't like it 2025-10-16 22:55:52 +02:00
6 changed files with 144 additions and 75 deletions
+15 -16
View File
@@ -25,14 +25,14 @@ jobs:
matrix: matrix:
include: include:
- os: ubuntu-latest - os: ubuntu-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-linux artifact_name: SoftwareManager-dev-linux
asset_name: SoftwareManager-dev-${{ github.sha }}-linux asset_name: SoftwareManager-dev-linux
- os: windows-latest - os: windows-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-windows.exe artifact_name: SoftwareManager-dev-windows.exe
asset_name: SoftwareManager-dev-${{ github.sha }}-windows.exe asset_name: SoftwareManager-dev-windows.exe
- os: macos-latest - os: macos-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-macos artifact_name: SoftwareManager-dev-macos
asset_name: SoftwareManager-dev-${{ github.sha }}-macos asset_name: SoftwareManager-dev-macos
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
@@ -121,23 +121,22 @@ jobs:
- name: Prepare release files - name: Prepare release files
run: | run: |
mkdir -p release mkdir -p release
SHORT_SHA="${{ steps.version.outputs.short_sha }}" cp artifacts/ubuntu-latest-build/SoftwareManager-dev-linux release/SoftwareManager-dev-linux
cp artifacts/ubuntu-latest-build/SoftwareManager-dev-*-linux release/SoftwareManager-dev-${SHORT_SHA}-linux cp artifacts/windows-latest-build/SoftwareManager-dev-windows.exe release/SoftwareManager-dev-windows.exe
cp artifacts/windows-latest-build/SoftwareManager-dev-*-windows.exe release/SoftwareManager-dev-${SHORT_SHA}-windows.exe cp artifacts/macos-latest-build/SoftwareManager-dev-macos release/SoftwareManager-dev-macos
cp artifacts/macos-latest-build/SoftwareManager-dev-*-macos release/SoftwareManager-dev-${SHORT_SHA}-macos chmod +x release/SoftwareManager-dev-linux release/SoftwareManager-dev-macos
chmod +x release/SoftwareManager-dev-*-linux release/SoftwareManager-dev-*-macos
ls -la release/ ls -la release/
- name: Create Prerelease (Public) - name: Create Release (Public)
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v2
with: with:
tag_name: ${{ steps.version.outputs.version }} tag_name: ${{ steps.version.outputs.version }}
name: ${{ steps.version.outputs.version }} name: ${{ steps.version.outputs.version }}
draft: false draft: false
prerelease: true prerelease: false
files: | files: |
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-linux release/SoftwareManager-dev-linux
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-windows.exe release/SoftwareManager-dev-windows.exe
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-macos release/SoftwareManager-dev-macos
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+41
View File
@@ -0,0 +1,41 @@
from bs4 import BeautifulSoup
import requests
def scrape_monkrus_telegram(query):
post = 100
max_posts = 250
posts: list[dict] = []
added = set()
while post <= max_posts:
url = f"https://t.me/s/real_monkrus/{post}"
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
if not bubbles:
break
for bubble in bubbles:
post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
if not post_txt or not post_txt.b:
continue
title = post_txt.b.text
link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
if query.lower() in title.lower() and link:
post_url = link["href"]
if post_url not in added:
added.add(post_url)
posts.append(dict(
author="m0nkrus",
id=len(posts) + 1,
title=title,
url=post_url
))
post += 22
posts.reverse()
return(posts)
+15 -24
View File
@@ -28,29 +28,23 @@ async def init_uztracker():
asyncio.run(init_uztracker()) asyncio.run(init_uztracker())
async def scrape_uztracker(search, max_results=450): def scrape_uztracker(search):
if up: if up:
search_url = url_uztracker + search
if state.debug:
print(search_url)
result = False result = False
global results global results
global resulttitles global resulttitles
results = [] results = []
resulttitles = [] resulttitles = []
per_page = 50
for start in range(0, max_results, per_page):
search_url = url_uztracker + search + f"&start={start}"
if state.debug:
print(search_url)
async with aiohttp.ClientSession() as session:
try: try:
async with session.get(search_url) as response:
response.raise_for_status()
text = await response.text()
soup = BeautifulSoup(text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
resultCount = 0 resultCount = 0
response = requests.get(search_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links: for link in links:
if link.b: if link.b:
resultCount += 1 resultCount += 1
@@ -58,25 +52,22 @@ async def scrape_uztracker(search, max_results=450):
resulttitles.append(link.b.text) resulttitles.append(link.b.text)
result = True result = True
if resultCount == 0: if result:
# add popup in gui for no result return resulttitles, results
break
except aiohttp.ClientError as e: if not result:
# add popup in gui for no result
return None
except requests.RequestException as e:
if result and state.debug: if result and state.debug:
print(f"Failed to fetch {search_url}: {e}") print(f"Failed to fetch {search_url}: {e}")
return None return None
else: else:
if state.debug: if state.debug:
print("Error: Uztracker down") print("Error: Uztracker down")
return None, None return None, None
if not results:
return None, None
return resulttitles, results
def get_post_title(post_url): def get_post_title(post_url):
if up: if up:
response = requests.get(post_url) response = requests.get(post_url)
+1 -1
View File
@@ -139,7 +139,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
toolbar.setLayoutDirection(Qt.RightToLeft) toolbar.setLayoutDirection(Qt.RightToLeft)
self.tracker_list = QComboBox() self.tracker_list = QComboBox()
self.tracker_list.addItems(["rutracker", "uztracker"]) self.tracker_list.addItems(["rutracker", "uztracker", "m0nkrus"])
self.tracker_list.activated.connect(self.set_tracker) self.tracker_list.activated.connect(self.set_tracker)
+43 -11
View File
@@ -1,6 +1,6 @@
import asyncio
from core.data.scrapers.uztracker import scrape_uztracker from core.data.scrapers.uztracker import scrape_uztracker
from core.data.scrapers.rutracker import scrape_rutracker from core.data.scrapers.rutracker import scrape_rutracker
from core.data.scrapers.monkrus import scrape_monkrus_telegram
from core.utils.network.jsonhandler import split_data, format_data from core.utils.network.jsonhandler import split_data, format_data
from core.utils.data.state import state from core.utils.data.state import state
@@ -12,22 +12,34 @@ def return_pressed(self):
return return
if state.debug: if state.debug:
print("User searched for:", search_text) print("User searched for:", search_text)
if state.tracker == "uztracker": if state.tracker == "uztracker":
response = asyncio.run(scrape_uztracker(search_text)) response = scrape_uztracker(search_text)
if state.tracker == "rutracker": if response:
state.post_titles, state.post_urls = response
self.softwareList.clear()
if state.post_titles and len(state.post_titles) > 0:
self.softwareList.addItems(state.post_titles)
self.show_empty_results(False)
self.post_author_list.clear()
else:
if state.debug:
print(f"No Results for {search_text}")
self.softwareList.clear()
self.show_empty_results(True)
else:
if state.debug:
print(f"No response from uztracker")
self.softwareList.clear()
self.show_empty_results(True)
elif state.tracker == "rutracker":
response = scrape_rutracker(search_text) response = scrape_rutracker(search_text)
if response: if response:
if state.tracker == "uztracker":
state.post_titles, state.post_urls, = response
self.softwareList.clear()
if state.post_titles:
self.softwareList.addItems(state.post_titles)
self.post_author_list.addItems(state.post_author)
if state.tracker == "rutracker":
_, state.posts, _, _, cached = split_data(response) _, state.posts, _, _, cached = split_data(response)
if state.posts == []: if state.posts == []:
if state.debug: if state.debug:
print(f"No Results found for \"{search_text}\"") print(f"No Results for {search_text}")
self.softwareList.clear() self.softwareList.clear()
self.show_empty_results(True) self.show_empty_results(True)
else: else:
@@ -39,4 +51,24 @@ def return_pressed(self):
self.softwareList.addItems(state.post_titles) self.softwareList.addItems(state.post_titles)
if state.debug == True: if state.debug == True:
print(f"Response Cached: {cached}") print(f"Response Cached: {cached}")
else:
if state.debug:
print(f"No response from rutracker")
self.softwareList.clear()
self.show_empty_results(True)
elif state.tracker == "m0nkrus":
state.posts = scrape_monkrus_telegram(search_text)
if state.posts == []:
if state.debug:
print(f"No Results for {search_text}")
self.softwareList.clear()
self.show_empty_results(True)
else:
state.post_titles, _, state.post_author = format_data(state.posts)
self.show_empty_results(False)
self.post_author_list.clear()
self.post_author_list.addItems(state.post_author)
self.softwareList.clear()
self.softwareList.addItems(state.post_titles)
+6
View File
@@ -19,6 +19,12 @@ def get_item_url(item, posts, post_titles): # softwarelist currentitem, post lis
if state.debug: if state.debug:
print("Found post URL: ", post_links[0]) print("Found post URL: ", post_links[0])
return post_links[0] return post_links[0]
if state.tracker == "m0nkrus":
item_dict = posts[post_index]
_, post_links, _ = format_data([item_dict])
if state.debug:
print("Found post URL: ", post_links[0])
return post_links[0]
return None return None