Compare commits

...

22 Commits

Author SHA1 Message Date
KeksNino c8a1e72107 remove test.txt 2025-10-18 22:09:22 +02:00
KeksNino 7c505f1402 this is a test 2025-10-18 16:45:09 +02:00
KeksNino 55e4f86a39 update workflows 2025-10-18 15:59:50 +02:00
KeksNino bd71b0e645 update workflows 2025-10-18 15:55:48 +02:00
KeksNino 2a86e4b774 update workflows 2025-10-18 15:50:16 +02:00
shayaa d17b4bd44a add & impl. m0nkrus scraperM 2025-10-18 02:32:33 +02:00
Vxrtrauter 10c935e852 fix tracker.py to handle m0nkrus scraper for downloads 2025-10-18 02:31:00 +02:00
Vxrtrauter bd8320a26c improve monkrus scraper for more results 2025-10-18 02:25:15 +02:00
Vxrtrauter c99ce339e0 fully impl. monkrus scraper (+ add search logic to scraper) 2025-10-18 01:49:00 +02:00
Vxrtrauter 1822a150d7 finnish monkrus scraper (probably) 2025-10-18 01:33:12 +02:00
Vxrtrauter 7e5e3033e6 add scraping logic 2025-10-18 01:22:11 +02:00
Vxrtrauter 1cab671a13 add more testing stuff 2025-10-18 01:08:36 +02:00
Vxrtrauter 5637e80657 added some more testing stuff 2025-10-17 21:23:17 +02:00
Vxrtrauter 2f78b812ba add besic scraping with message retrieving logic; still featuring full testing & debug outpus 2025-10-17 20:23:44 +02:00
Vxrtrauter 5af9e79ecc Refractor searchhelper to finally be readable, reverd uztracker scraper to old format since i didn't like it 2025-10-16 22:55:52 +02:00
shayaa 4dadc9ba50 Add debug feature 2025-10-16 15:55:39 +02:00
Vxrtrauter 719c4e86e8 improved cache debug 2025-10-16 15:53:05 +02:00
Vxrtrauter 999c8e9ba4 improved debug output 2025-10-16 15:49:00 +02:00
Vxrtrauter 3f6889da56 improve debug messages 2025-10-16 15:40:43 +02:00
Vxrtrauter 69ac739be6 add debug arg 2025-10-16 15:36:10 +02:00
shayaa 141cc8fadc Update README.md 2025-10-14 23:13:14 +02:00
shayaa 80c6ece72e Update README.md 2025-10-14 23:12:34 +02:00
13 changed files with 187 additions and 86 deletions
+15 -16
View File
@@ -25,14 +25,14 @@ jobs:
matrix:
include:
- os: ubuntu-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-linux
asset_name: SoftwareManager-dev-${{ github.sha }}-linux
artifact_name: SoftwareManager-dev-linux
asset_name: SoftwareManager-dev-linux
- os: windows-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-windows.exe
asset_name: SoftwareManager-dev-${{ github.sha }}-windows.exe
artifact_name: SoftwareManager-dev-windows.exe
asset_name: SoftwareManager-dev-windows.exe
- os: macos-latest
artifact_name: SoftwareManager-dev-${{ github.sha }}-macos
asset_name: SoftwareManager-dev-${{ github.sha }}-macos
artifact_name: SoftwareManager-dev-macos
asset_name: SoftwareManager-dev-macos
runs-on: ${{ matrix.os }}
@@ -121,23 +121,22 @@ jobs:
- name: Prepare release files
run: |
mkdir -p release
SHORT_SHA="${{ steps.version.outputs.short_sha }}"
cp artifacts/ubuntu-latest-build/SoftwareManager-dev-*-linux release/SoftwareManager-dev-${SHORT_SHA}-linux
cp artifacts/windows-latest-build/SoftwareManager-dev-*-windows.exe release/SoftwareManager-dev-${SHORT_SHA}-windows.exe
cp artifacts/macos-latest-build/SoftwareManager-dev-*-macos release/SoftwareManager-dev-${SHORT_SHA}-macos
chmod +x release/SoftwareManager-dev-*-linux release/SoftwareManager-dev-*-macos
cp artifacts/ubuntu-latest-build/SoftwareManager-dev-linux release/SoftwareManager-dev-linux
cp artifacts/windows-latest-build/SoftwareManager-dev-windows.exe release/SoftwareManager-dev-windows.exe
cp artifacts/macos-latest-build/SoftwareManager-dev-macos release/SoftwareManager-dev-macos
chmod +x release/SoftwareManager-dev-linux release/SoftwareManager-dev-macos
ls -la release/
- name: Create Prerelease (Public)
- name: Create Release (Public)
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ steps.version.outputs.version }}
name: ${{ steps.version.outputs.version }}
draft: false
prerelease: true
prerelease: false
files: |
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-linux
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-windows.exe
release/SoftwareManager-dev-${{ steps.version.outputs.short_sha }}-macos
release/SoftwareManager-dev-linux
release/SoftwareManager-dev-windows.exe
release/SoftwareManager-dev-macos
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+11
View File
@@ -27,6 +27,17 @@ SoftwareManager is a Python-based GUI/TUI tool that simplifies downloading and m
- If you want to host your own server, clone the server branch of this repo, enter your cookie for rutracker.org and run server.py. Detailed Instructions on how to get your rutracker cookie are in the README of its branch.
- This tool is for educational purposes only. Use it responsibly.
## Star History
<a href="https://www.star-history.com/#KeksPirates/SoftwareManager&type=date&legend=top-left">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=KeksPirates/SoftwareManager&type=date&theme=dark&legend=top-left" />
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=KeksPirates/SoftwareManager&type=date&legend=top-left" />
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=KeksPirates/SoftwareManager&type=date&legend=top-left" />
</picture>
</a>
---
**Disclaimer:** SoftwareManager is intended for legal and ethical use only. Ensure compliance with applicable laws and regulations when using this tool.
+41
View File
@@ -0,0 +1,41 @@
from bs4 import BeautifulSoup
import requests
def scrape_monkrus_telegram(query):
post = 100
max_posts = 250
posts: list[dict] = []
added = set()
while post <= max_posts:
url = f"https://t.me/s/real_monkrus/{post}"
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
bubbles = soup.find_all("div", class_="tgme_widget_message_bubble")
if not bubbles:
break
for bubble in bubbles:
post_txt = bubble.find("div", class_="tgme_widget_message_text js-message_text")
if not post_txt or not post_txt.b:
continue
title = post_txt.b.text
link = bubble.find("a", href=lambda x: x and x.startswith("https://uztracker.net"))
if query.lower() in title.lower() and link:
post_url = link["href"]
if post_url not in added:
added.add(post_url)
posts.append(dict(
author="m0nkrus",
id=len(posts) + 1,
title=title,
url=post_url
))
post += 22
posts.reverse()
return(posts)
+2
View File
@@ -4,6 +4,8 @@ from core.utils.data.state import state
def scrape_rutracker(search_text):
search = requests.get(f"{state.api_url}/search/{search_text}")
if state.debug:
print("Sent request to server")
if search:
try:
return search.text
+37 -42
View File
@@ -17,62 +17,56 @@ async def init_uztracker():
up = True
else:
up = False
print(f"Uztracker seems down, status code: {response.status_code}")
if state.debug:
print(f"Uztracker seems down, status code: {response.status_code}")
except requests.exceptions.RequestException as e:
print(f"\nRequest Exception on {url_uztracker}:")
print(e)
print("\nIs the Site down?")
if state.debug:
print(f"\nRequest Exception on {url_uztracker}:")
print(e)
print("\nIs the Site down?")
up = False
asyncio.run(init_uztracker())
async def scrape_uztracker(search, max_results=450):
def scrape_uztracker(search):
if up:
search_url = url_uztracker + search
if state.debug:
print(search_url)
result = False
global results
global resulttitles
results = []
results = []
resulttitles = []
per_page = 50
try:
resultCount = 0
response = requests.get(search_url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
for link in links:
if link.b:
resultCount += 1
results.append(link['href'])
resulttitles.append(link.b.text)
result = True
for start in range(0, max_results, per_page):
search_url = url_uztracker + search + f"&start={start}"
if state.debug:
print(search_url)
if result:
return resulttitles, results
async with aiohttp.ClientSession() as session:
try:
async with session.get(search_url) as response:
response.raise_for_status()
text = await response.text()
soup = BeautifulSoup(text, 'html.parser')
links = soup.find_all('a', class_="genmed tLink", href=lambda x: x and x.startswith('./viewtopic'))
resultCount = 0
for link in links:
if link.b:
resultCount += 1
results.append(link['href'])
resulttitles.append(link.b.text)
result = True
if resultCount == 0:
# add popup in gui for no result
break
except aiohttp.ClientError as e:
if result:
print(f"Failed to fetch {search_url}: {e}")
return None
if not result:
# add popup in gui for no result
return None
else:
except requests.RequestException as e:
if result and state.debug:
print(f"Failed to fetch {search_url}: {e}")
return None
else:
if state.debug:
print("Error: Uztracker down")
return None, None
if not results:
return None, None
return resulttitles, results
return None, None
def get_post_title(post_url):
if up:
@@ -85,7 +79,8 @@ def get_post_title(post_url):
if state.debug:
print("Program not found")
else:
print("Error: Uztracker down")
if state.debug:
print("Error: Uztracker down")
return None
-1
View File
@@ -114,7 +114,6 @@ def settings_dialog(self):
save_btn.clicked.connect(lambda: save_settings(thread_box.value(), close_settings, api_url.text(), download_path.text(), speed_limit.value()))
cancel_btn.clicked.connect(dialog.reject)
print(thread_box.value())
layout.addWidget(cancel_btn)
layout.addWidget(save_btn)
+1 -1
View File
@@ -139,7 +139,7 @@ class MainWindow(QtWidgets.QMainWindow, QWidget):
toolbar.setLayoutDirection(Qt.RightToLeft)
self.tracker_list = QComboBox()
self.tracker_list.addItems(["rutracker", "uztracker"])
self.tracker_list.addItems(["rutracker", "uztracker", "m0nkrus"])
self.tracker_list.activated.connect(self.set_tracker)
+52 -20
View File
@@ -1,33 +1,45 @@
import asyncio
from core.data.scrapers.uztracker import scrape_uztracker
from core.data.scrapers.rutracker import scrape_rutracker
from core.data.scrapers.monkrus import scrape_monkrus_telegram
from core.utils.network.jsonhandler import split_data, format_data
from core.utils.data.state import state
def return_pressed(self):
search_text = self.searchbar.text()
if search_text == "":
if state.debug:
print("Error: Can't search for nothing")
return
search_text = self.searchbar.text()
if search_text == "":
if state.debug:
print("User searched for:", search_text)
if state.tracker == "uztracker":
response = asyncio.run(scrape_uztracker(search_text))
if state.tracker == "rutracker":
response = scrape_rutracker(search_text)
print("Error: Can't search for nothing")
return
if state.debug:
print("User searched for:", search_text)
if state.tracker == "uztracker":
response = scrape_uztracker(search_text)
if response:
if state.tracker == "uztracker":
state.post_titles, state.post_urls, = response
state.post_titles, state.post_urls = response
self.softwareList.clear()
if state.post_titles and len(state.post_titles) > 0:
self.softwareList.addItems(state.post_titles)
self.show_empty_results(False)
self.post_author_list.clear()
else:
if state.debug:
print(f"No Results for {search_text}")
self.softwareList.clear()
if state.post_titles:
self.softwareList.addItems(state.post_titles)
self.post_author_list.addItems(state.post_author)
if state.tracker == "rutracker":
_, state.posts, _, _, cached = split_data(response)
self.show_empty_results(True)
else:
if state.debug:
print(f"No response from uztracker")
self.softwareList.clear()
self.show_empty_results(True)
elif state.tracker == "rutracker":
response = scrape_rutracker(search_text)
if response:
_, state.posts, _, _, cached = split_data(response)
if state.posts == []:
if state.debug:
print(f"No Results found for \"{search_text}\"")
print(f"No Results for {search_text}")
self.softwareList.clear()
self.show_empty_results(True)
else:
@@ -38,5 +50,25 @@ def return_pressed(self):
self.softwareList.clear()
self.softwareList.addItems(state.post_titles)
if state.debug == True:
print(f"Cached: {cached}")
print(f"Response Cached: {cached}")
else:
if state.debug:
print(f"No response from rutracker")
self.softwareList.clear()
self.show_empty_results(True)
elif state.tracker == "m0nkrus":
state.posts = scrape_monkrus_telegram(search_text)
if state.posts == []:
if state.debug:
print(f"No Results for {search_text}")
self.softwareList.clear()
self.show_empty_results(True)
else:
state.post_titles, _, state.post_author = format_data(state.posts)
self.show_empty_results(False)
self.post_author_list.clear()
self.post_author_list.addItems(state.post_author)
self.softwareList.clear()
self.softwareList.addItems(state.post_titles)
+5
View File
@@ -1,11 +1,16 @@
from .aria2_integration import run_aria2p
from core.utils.data.state import state
def start_client():
global aria2
aria2 = run_aria2p()
if state.debug:
print("Started Aria2p")
def add_magnet(uri):
aria2.add_magnet(uri)
if state.debug:
print("Magnet URI added to Aria2")
+14 -1
View File
@@ -5,16 +5,27 @@ from core.utils.network.jsonhandler import format_data
def get_item_url(item, posts, post_titles): # oftwarelist currentitem, post list (dict), post titles list
def get_item_url(item, posts, post_titles): # softwarelist currentitem, post list (dict), post titles list
post_index = post_titles.index(item)
if 0 <= post_index < len(post_titles):
if state.tracker == "uztracker":
item = "https://uztracker.net/" + state.post_urls[post_index].lstrip("./")
if state.debug:
print("Found post URL: ", item)
return item
if state.tracker == "rutracker":
item_dict = posts[post_index]
_, post_links, _ = format_data([item_dict])
if state.debug:
print("Found post URL: ", post_links[0])
return post_links[0]
if state.tracker == "m0nkrus":
item_dict = posts[post_index]
_, post_links, _ = format_data([item_dict])
if state.debug:
print("Found post URL: ", post_links[0])
return post_links[0]
return None
@@ -22,6 +33,8 @@ def get_item_url(item, posts, post_titles): # oftwarelist currentitem, post list
def get_magnet_link(post_url):
try:
response = requests.get(post_url) # eventually impl. cloudscraper
if state.debug:
print("Sent Request to retrieve Magnet Link...")
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
magnet_link = soup.find('a', href=lambda x: x and x.startswith('magnet:'))
+1 -1
View File
@@ -22,7 +22,7 @@ def kill_aria2server():
if proc.name() == process:
proc.kill()
if state.debug:
print("Killed Aria2c")
print(f"Killed Aria2c (PID {proc.pid})")
except psutil.NoSuchProcess:
pass
+1 -1
View File
@@ -10,7 +10,7 @@ import threading
def download_selected(item, posts, post_titles):
if item is not None:
if state.debug:
print(f"network {item.text()}")
print(f"Downloading {item.text()}")
run_thread(threading.Thread(target=run_download, args=(item.text(), posts, post_titles)))
else:
+7 -3
View File
@@ -10,11 +10,14 @@ import qdarktheme
import darkdetect
import threading
import signal
import atexit
import argparse
import sys
# Debug Output
state.debug = True
parser = argparse.ArgumentParser()
parser.add_argument("--debug", action="store_true")
args = parser.parse_args()
def run_gui():
@@ -36,6 +39,7 @@ def keyboardinterrupthandler(signum, frame):
if __name__ == "__main__":
read_config()
state.debug = args.debug # override of read_config
if state.debug:
print("Starting Aria2 Server")
state.aria2process = run_aria2server()