Rewrite
This commit is contained in:
7x11x13
@@ -1,73 +1,8 @@
|
||||
import atexit
|
||||
import importlib.metadata
|
||||
import logging
|
||||
import os
|
||||
from configparser import ConfigParser
|
||||
|
||||
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO'))
|
||||
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
__version__ = 'v0.0.7'
|
||||
|
||||
if 'XDG_CONFIG_HOME' in os.environ:
|
||||
config_dir = os.path.join(
|
||||
os.environ['XDG_CONFIG_HOME'], 'free-bandcamp-downloader')
|
||||
else:
|
||||
config_dir = os.path.join(os.path.expanduser(
|
||||
'~'), '.config', 'free-bandcamp-downloader')
|
||||
|
||||
if 'XDG_DATA_HOME' in os.environ:
|
||||
data_dir = os.path.join(
|
||||
os.environ['XDG_DATA_HOME'], 'free-bandcamp-downloader')
|
||||
else:
|
||||
data_dir = os.path.join(os.path.expanduser(
|
||||
'~'), '.local', 'share', 'free-bandcamp-downloader')
|
||||
|
||||
download_history_file = os.path.join(data_dir, 'downloaded.txt')
|
||||
|
||||
default_config = \
|
||||
f"""[free-bandcamp-downloader]
|
||||
country = United States
|
||||
zipcode = 00000
|
||||
email = auto
|
||||
format = FLAC
|
||||
dir = .
|
||||
download_history_file = {download_history_file}"""
|
||||
|
||||
config_file = os.path.join(config_dir, 'free-bandcamp-downloader.cfg')
|
||||
|
||||
if not os.path.exists(config_file):
|
||||
if not os.path.exists(config_dir):
|
||||
os.makedirs(config_dir)
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(default_config)
|
||||
|
||||
if not os.path.exists(download_history_file):
|
||||
if not os.path.exists(data_dir):
|
||||
os.makedirs(data_dir)
|
||||
with open(download_history_file, 'w') as f:
|
||||
pass
|
||||
|
||||
parser = ConfigParser()
|
||||
parser.read(config_file)
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, parser: ConfigParser):
|
||||
self.parser = parser
|
||||
atexit.register(self.save)
|
||||
|
||||
def get(self, key):
|
||||
return self.parser['free-bandcamp-downloader'].get(key, None)
|
||||
|
||||
def set(self, key, value):
|
||||
self.parser['free-bandcamp-downloader'][key] = value
|
||||
|
||||
def save(self):
|
||||
with open(config_file, 'w') as f:
|
||||
self.parser.write(f)
|
||||
|
||||
def __str__(self):
|
||||
return str(dict(self.parser['free-bandcamp-downloader']))
|
||||
|
||||
|
||||
config = Config(parser)
|
||||
__version__ = importlib.metadata.version("free-bandcamp-downloader")
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
"""Download free albums and tracks from Bandcamp
|
||||
Usage:
|
||||
bcdl-free (-a <URL> | -l <URL>)[--force][--no-unzip][-d | --dir <dir>][-e | --email <email>]
|
||||
[-z | --zipcode <zipcode>][-c | --country <country>][-f | --format <format>]
|
||||
bcdl-free setdefault [-d | --dir <dir>][-e | --email <email>][-z | --zipcode <zipcode>]
|
||||
[-c | --country <country>][-f | --format <format>]
|
||||
bcdl-free defaults
|
||||
bcdl-free clear
|
||||
bcdl-free (-h | --help)
|
||||
bcdl-free --version
|
||||
Options:
|
||||
-h --help Show this screen
|
||||
--version Show version
|
||||
-a <URL> Download the album at URL
|
||||
-l <URL> Download all free albums of the label at URL
|
||||
--force Download even if album has been downloaded before
|
||||
--no-unzip Don't unzip downloaded albums
|
||||
setdefault Set default options
|
||||
defaults List the default options
|
||||
clear Clear download history
|
||||
-d --dir <dir> Set download directory
|
||||
-c --country <country> Set country
|
||||
-z --zipcode <zipcode> Set zipcode
|
||||
-e --email <email> Set email (set to 'auto' to automatically download from a disposable email)
|
||||
-f --format <format> Set format
|
||||
Formats:
|
||||
- FLAC
|
||||
- V0MP3
|
||||
- 320MP3
|
||||
- AAC
|
||||
- Ogg
|
||||
- ALAC
|
||||
- WAV
|
||||
- AIFF
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import dataclasses
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pprint
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from configparser import ConfigParser
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Set
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import mutagen
|
||||
import pyrfc6266
|
||||
import requests
|
||||
import secmail
|
||||
from bs4 import BeautifulSoup
|
||||
from docopt import docopt
|
||||
from tqdm import tqdm
|
||||
|
||||
from free_bandcamp_downloader import __version__, logger
|
||||
|
||||
|
||||
@dataclass
|
||||
class BCFreeDownloaderOptions:
|
||||
country: str = None
|
||||
zipcode: str = None
|
||||
email: str = None
|
||||
format: str = None
|
||||
dir: str = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class BCFreeDownloaderAlbumData:
|
||||
about: str = None
|
||||
credits: str = None
|
||||
tags: str = None
|
||||
|
||||
|
||||
class BCFreeDownloadError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BCFreeDownloader:
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
LINK_REGEX = re.compile(r'<a href="(?P<url>[^"]*)">')
|
||||
RETRY_URL_REGEX = re.compile(r'"retry_url":"(?P<retry_url>[^"]*)"')
|
||||
FORMATS = {
|
||||
"FLAC": "flac",
|
||||
"V0MP3": "mp3-v0",
|
||||
"320MP3": "mp3-320",
|
||||
"AAC": "aac-hi",
|
||||
"Ogg": "vorbis",
|
||||
"ALAC": "alac",
|
||||
"WAV": "wav",
|
||||
"AIFF": "aiff-lossless",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
options: BCFreeDownloaderOptions,
|
||||
config_dir: str,
|
||||
download_history_file: str,
|
||||
unzip: bool = True,
|
||||
):
|
||||
self.options = options
|
||||
self.config_dir = config_dir
|
||||
self.download_history_file = download_history_file
|
||||
self.downloaded: Set[str] = set()
|
||||
self.mail_session = None
|
||||
self.mail_album_data: Dict[str, BCFreeDownloaderAlbumData] = {}
|
||||
self.unzip = unzip
|
||||
self.session = None
|
||||
self._init_email()
|
||||
self._init_downloaded()
|
||||
self._init_session()
|
||||
|
||||
def _init_email(self):
|
||||
if not self.options.email or self.options.email == "auto":
|
||||
self.mail_session = secmail.Client(self.config_dir)
|
||||
self.options.email = self.mail_session.random_email(1, "1secmail.com")[0]
|
||||
|
||||
def _init_downloaded(self):
|
||||
if self.download_history_file:
|
||||
with open(self.download_history_file, "r") as f:
|
||||
for line in f:
|
||||
self.downloaded.add(line.strip())
|
||||
|
||||
def _init_session(self):
|
||||
self.session = requests.Session()
|
||||
|
||||
def _download_file(
|
||||
self,
|
||||
download_page_url: str,
|
||||
format: str,
|
||||
album_data: BCFreeDownloaderAlbumData = None,
|
||||
) -> str:
|
||||
r = self.session.get(download_page_url)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text, "html.parser")
|
||||
album_url = soup.find("div", class_="download-artwork").find("a").attrs["href"]
|
||||
if album_data is None:
|
||||
album_data = self.mail_album_data[album_url]
|
||||
|
||||
data = json.loads(soup.find("div", {"id": "pagedata"}).attrs["data-blob"])
|
||||
download_url = data["digital_items"][0]["downloads"][self.FORMATS[format]][
|
||||
"url"
|
||||
]
|
||||
|
||||
def download(download_url: str) -> str:
|
||||
with self.session.get(download_url, stream=True) as r:
|
||||
r.raise_for_status()
|
||||
size = int(r.headers["content-length"])
|
||||
name = pyrfc6266.requests_response_to_filename(r)
|
||||
file_name = os.path.join(self.options.dir, name)
|
||||
with tqdm(total=size, unit="iB", unit_scale=True) as pbar:
|
||||
with open(file_name, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
return file_name
|
||||
|
||||
try:
|
||||
file_name = download(download_url)
|
||||
except:
|
||||
statdownload_url = download_url.replace("/download/", "/statdownload/")
|
||||
with self.session.get(statdownload_url) as r:
|
||||
r.raise_for_status()
|
||||
download_url = self.RETRY_URL_REGEX.search(r.text).group("retry_url")
|
||||
file_name = download(download_url)
|
||||
|
||||
logger.info(f"Downloaded {file_name}")
|
||||
|
||||
if file_name.endswith("zip") and self.unzip:
|
||||
# Unzip archive
|
||||
dir_name = file_name[:-4]
|
||||
with zipfile.ZipFile(file_name, "r") as f:
|
||||
f.extractall(dir_name)
|
||||
logger.info(f"Unzipped to {dir_name}. Use --no-unzip to prevent this")
|
||||
os.remove(file_name)
|
||||
files = glob.glob(os.path.join(dir_name, "*"))
|
||||
else:
|
||||
files = [file_name]
|
||||
# Tag downloaded audio files with url & comment
|
||||
logger.info("Setting tags...")
|
||||
for file in files:
|
||||
f = mutagen.File(file)
|
||||
if f is None:
|
||||
continue
|
||||
f["website"] = album_url
|
||||
if album_data.tags:
|
||||
f["genre"] = album_data.tags
|
||||
comment = ""
|
||||
if album_data.about:
|
||||
comment += album_data.about
|
||||
if album_data.about and album_data.credits:
|
||||
comment += "\n\n"
|
||||
if album_data.credits:
|
||||
comment += album_data.credits
|
||||
f["comment"] = comment
|
||||
f.save()
|
||||
# successfully downloaded file, add to download history
|
||||
self.downloaded.add(album_url)
|
||||
if self.download_history_file:
|
||||
with open(self.download_history_file, "a") as f:
|
||||
f.write(f"{album_url}\n")
|
||||
|
||||
return album_url
|
||||
|
||||
@staticmethod
|
||||
def _get_album_data_from_soup(soup: BeautifulSoup) -> BCFreeDownloaderAlbumData:
|
||||
album_data = BCFreeDownloaderAlbumData()
|
||||
album_data.about = soup.find("div", class_="tralbum-about").get_text("\n")
|
||||
album_data.credits = soup.find("div", class_="tralbum-credits").get_text("\n")
|
||||
tags = [tag.get_text() for tag in soup.find_all("a", class_="tag")]
|
||||
album_data.tags = ",".join(sorted(tags))
|
||||
return album_data
|
||||
|
||||
def download_album(self, url: str, force: bool = False):
|
||||
# Remove url params
|
||||
url = urlsplit(url).geturl()
|
||||
if url in self.downloaded and not force:
|
||||
raise BCFreeDownloadError(
|
||||
f"{url} already downloaded. To download anyways, use option --force"
|
||||
)
|
||||
r = self.session.get(url)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text, "html.parser")
|
||||
album_data = self._get_album_data_from_soup(soup)
|
||||
|
||||
logger.debug(f"Album data: {album_data}")
|
||||
|
||||
tralbum_data = soup.find("script", {"data-tralbum": True}).attrs["data-tralbum"]
|
||||
tralbum_data = json.loads(tralbum_data)
|
||||
|
||||
if tralbum_data["current"]["minimum_price"] == 0:
|
||||
if tralbum_data["current"]["require_email"]:
|
||||
logger.info(f"{url} requires email")
|
||||
email_post_url = urljoin(url, "/email_download")
|
||||
r = self.session.post(
|
||||
email_post_url,
|
||||
data={
|
||||
"encoding_name": "none",
|
||||
"item_id": tralbum_data["current"]["id"],
|
||||
"item_type": tralbum_data["current"]["type"],
|
||||
"address": self.options.email,
|
||||
"country": self.options.country,
|
||||
"postcode": self.options.zipcode,
|
||||
},
|
||||
)
|
||||
r.raise_for_status()
|
||||
r = r.json()
|
||||
if not r["ok"]:
|
||||
raise ValueError(f"Bad response when sending email address: {r}")
|
||||
self.mail_album_data[url] = album_data
|
||||
else:
|
||||
logger.info(f"{url} does not require email")
|
||||
self._download_file(
|
||||
tralbum_data["freeDownloadPage"], self.options.format, album_data
|
||||
)
|
||||
else:
|
||||
raise BCFreeDownloadError(f"{url} is not free")
|
||||
|
||||
def download_label(self, url: str, force: bool = False):
|
||||
r = self.session.get(url)
|
||||
r.raise_for_status()
|
||||
soup = BeautifulSoup(r.text)
|
||||
for album_title in soup.find_all("p", class_="title"):
|
||||
album_link = album_title.parent.attrs["href"]
|
||||
logger.info(f"Downloading {album_link}")
|
||||
try:
|
||||
self.download_album(album_link, force)
|
||||
except BCFreeDownloadError as ex:
|
||||
logger.info(ex)
|
||||
|
||||
def wait_for_email_downloads(self):
|
||||
checked_ids = set()
|
||||
while (expected_emails := len(self.mail_album_data)) > 0:
|
||||
logger.info(f"Waiting for {expected_emails} emails from Bandcamp...")
|
||||
time.sleep(5)
|
||||
for email in self.mail_session.get_inbox(self.options.email):
|
||||
if email.id not in checked_ids:
|
||||
checked_ids.add(email.id)
|
||||
if (
|
||||
email.from_address.endswith("@email.bandcamp.com")
|
||||
and "download" in email.subject
|
||||
):
|
||||
logger.info(f'Received email "{email.subject}"')
|
||||
email = self.mail_session.get_message(
|
||||
self.options.email, email.id
|
||||
)
|
||||
match = self.LINK_REGEX.search(email.html_body)
|
||||
if match:
|
||||
download_url = match.group("url")
|
||||
album_url = self._download_file(
|
||||
download_url, self.options.format
|
||||
)
|
||||
self.mail_album_data.pop(album_url)
|
||||
|
||||
|
||||
class BCFreeDownloaderConfig:
|
||||
def __init__(self, config_path: str):
|
||||
self.config_path = config_path
|
||||
self.parser = ConfigParser()
|
||||
self.parser.read(config_path)
|
||||
atexit.register(self.save)
|
||||
|
||||
def get(self, key):
|
||||
return self.parser["free-bandcamp-downloader"].get(key, None)
|
||||
|
||||
def set(self, key, value):
|
||||
self.parser["free-bandcamp-downloader"][key] = value
|
||||
|
||||
def save(self):
|
||||
with open(self.config_path, "w") as f:
|
||||
self.parser.write(f)
|
||||
|
||||
def __str__(self):
|
||||
return pprint.pformat(dict(self.parser["free-bandcamp-downloader"]), indent=2)
|
||||
|
||||
|
||||
def get_config_dir():
|
||||
if "XDG_CONFIG_HOME" in os.environ:
|
||||
config_dir = os.path.join(
|
||||
os.environ["XDG_CONFIG_HOME"], "free-bandcamp-downloader"
|
||||
)
|
||||
else:
|
||||
config_dir = os.path.join(
|
||||
os.path.expanduser("~"), ".config", "free-bandcamp-downloader"
|
||||
)
|
||||
return config_dir
|
||||
|
||||
|
||||
def get_data_dir():
|
||||
if "XDG_DATA_HOME" in os.environ:
|
||||
data_dir = os.path.join(os.environ["XDG_DATA_HOME"], "free-bandcamp-downloader")
|
||||
else:
|
||||
data_dir = os.path.join(
|
||||
os.path.expanduser("~"), ".local", "share", "free-bandcamp-downloader"
|
||||
)
|
||||
return data_dir
|
||||
|
||||
|
||||
def get_config(data_dir: str, config_dir: str):
|
||||
download_history_file = os.path.join(data_dir, "downloaded.txt")
|
||||
default_config = f"""[free-bandcamp-downloader]
|
||||
country = United States
|
||||
zipcode = 00000
|
||||
email = auto
|
||||
format = FLAC
|
||||
dir = .
|
||||
download_history_file = {download_history_file}"""
|
||||
config_file = os.path.join(config_dir, "free-bandcamp-downloader.cfg")
|
||||
if not os.path.exists(config_file):
|
||||
if not os.path.exists(config_dir):
|
||||
os.makedirs(config_dir)
|
||||
with open(config_file, "w") as f:
|
||||
f.write(default_config)
|
||||
if not os.path.exists(download_history_file):
|
||||
if not os.path.exists(data_dir):
|
||||
os.makedirs(data_dir)
|
||||
with open(download_history_file, "w") as f:
|
||||
pass
|
||||
config = BCFreeDownloaderConfig(config_file)
|
||||
return config
|
||||
|
||||
|
||||
def main():
|
||||
data_dir = get_data_dir()
|
||||
config_dir = get_config_dir()
|
||||
config = get_config(data_dir, config_dir)
|
||||
options = BCFreeDownloaderOptions()
|
||||
arguments = docopt(__doc__, version=__version__)
|
||||
if arguments["-a"] or arguments["-l"] or arguments["setdefault"]:
|
||||
# set options
|
||||
for field in dataclasses.fields(options):
|
||||
option = field.name
|
||||
arg = f"--{option}"
|
||||
if arguments[arg]:
|
||||
setattr(options, option, arguments[arg][0])
|
||||
else:
|
||||
setattr(options, option, config.get(option))
|
||||
if not getattr(options, option):
|
||||
logger.error(
|
||||
f'{option} is not set, use "bcdl-free setdefault {arg} <{option}>"'
|
||||
)
|
||||
sys.exit(1)
|
||||
if options.format not in BCFreeDownloader.FORMATS:
|
||||
logger.error(
|
||||
f'{options["format"]} is not a valid format. See "bcdl-free -h" for valid formats'
|
||||
)
|
||||
sys.exit(1)
|
||||
if arguments["-a"] or arguments["-l"]:
|
||||
# init downloader
|
||||
downloader = BCFreeDownloader(
|
||||
options,
|
||||
config_dir,
|
||||
config.get("download_history_file"),
|
||||
not arguments["--no-unzip"],
|
||||
)
|
||||
if arguments["-a"]:
|
||||
downloader.download_album(arguments["-a"], arguments["--force"])
|
||||
elif arguments["-l"]:
|
||||
downloader.download_label(arguments["-l"], arguments["--force"])
|
||||
# finish up downloading
|
||||
downloader.wait_for_email_downloads()
|
||||
elif arguments["setdefault"]:
|
||||
# write arguments to config
|
||||
for field in dataclasses.fields(options):
|
||||
option = field.name
|
||||
arg = f"--{option}"
|
||||
if arguments[arg]:
|
||||
config.set(option, arguments[arg][0])
|
||||
elif arguments["defaults"]:
|
||||
print(str(config))
|
||||
elif arguments["clear"]:
|
||||
with open(config.get("download_history_file"), "w"):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,390 +0,0 @@
|
||||
"""Download free albums and tracks from Bandcamp
|
||||
Usage:
|
||||
bcdl-free (-a <URL> | -l <URL>)[--force][--no-unzip][-d | --dir <dir>][-e | --email <email>]
|
||||
[-z | --zipcode <zipcode>][-c | --country <country>][-f | --format <format>]
|
||||
bcdl-free setdefault [-d | --dir <dir>][-e | --email <email>][-z | --zipcode <zipcode>]
|
||||
[-c | --country <country>][-f | --format <format>]
|
||||
bcdl-free defaults
|
||||
bcdl-free clear
|
||||
bcdl-free (-h | --help)
|
||||
bcdl-free --version
|
||||
Options:
|
||||
-h --help Show this screen
|
||||
--version Show version
|
||||
-a <URL> Download the album at URL
|
||||
-l <URL> Download all free albums of the label at URL
|
||||
--force Download even if album has been downloaded before
|
||||
--no-unzip Don't unzip downloaded albums
|
||||
setdefault Set default options
|
||||
defaults List the default options
|
||||
clear Clear download history
|
||||
-d --dir <dir> Set download directory
|
||||
-c --country <country> Set country
|
||||
-z --zipcode <zipcode> Set zipcode
|
||||
-e --email <email> Set email (set to 'auto' to automatically download from a disposable email)
|
||||
-f --format <format> Set format
|
||||
Formats:
|
||||
- FLAC
|
||||
- V0MP3
|
||||
- 320MP3
|
||||
- AAC
|
||||
- Ogg
|
||||
- ALAC
|
||||
- WAV
|
||||
- AIFF
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import bs4
|
||||
import mutagen
|
||||
from bs4 import BeautifulSoup, SoupStrainer
|
||||
from docopt import docopt
|
||||
from guerrillamail import GuerrillaMailSession
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.firefox.options import Options
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import Select
|
||||
from selenium.webdriver.support.wait import WebDriverWait
|
||||
|
||||
from free_bandcamp_downloader import __version__, config, logger
|
||||
|
||||
# Global variables
|
||||
arguments = None
|
||||
mail_session = None
|
||||
expected_emails = 0
|
||||
mail_album_data = {}
|
||||
downloaded = set()
|
||||
options = {
|
||||
'country': None,
|
||||
'zipcode': None,
|
||||
'email': None,
|
||||
'format': None,
|
||||
'dir': None
|
||||
}
|
||||
|
||||
# Constants
|
||||
|
||||
link_regex = re.compile('<a href="(?P<url>http[^"]*)">')
|
||||
|
||||
xpath = {
|
||||
'buy': '//*[@class="ft compound-button main-button"]/button[@class="download-link buy-link"]',
|
||||
'price': '//input[@id="userPrice"]',
|
||||
'download-nyp': '//a[@class="download-panel-free-download-link"]',
|
||||
'checkout': '//button[@class="download-panel-checkout-button" and not(ancestor::div[contains(@style,"display:none")])]',
|
||||
'email': '//input[@id="fan_email_address"]',
|
||||
'country': '//select[@id="fan_email_country"]',
|
||||
'zipcode': '//input[@id="fan_email_postalcode"]',
|
||||
'preparing': '//span[@class="preparing-title"]',
|
||||
'formats': '//select[@id="format-type"]',
|
||||
'download': '//div[@class="download-format-tmp"]/a[1]',
|
||||
'albums': '//a[./p[@class="title"]]',
|
||||
'album-link': '//div[@class="download-artwork"]/a',
|
||||
'album-tags': '//div[contains(@class, "tralbum-tags-nu")]',
|
||||
'album-credits': '//div[@class="tralbumData tralbum-credits"]',
|
||||
'album-about': '//div[@class="tralbumData tralbum-about"]',
|
||||
}
|
||||
|
||||
formats = {
|
||||
'FLAC': 'flac',
|
||||
'V0MP3': 'mp3-v0',
|
||||
'320MP3': 'mp3-320',
|
||||
'AAC': 'aac-hi',
|
||||
'Ogg': 'vorbis',
|
||||
'ALAC': 'alac',
|
||||
'WAV': 'wav',
|
||||
'AIFF': 'aiff-lossless'
|
||||
}
|
||||
|
||||
|
||||
def wait():
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def init_email():
|
||||
global mail_session
|
||||
if not mail_session and (not options['email'] or options['email'] == 'auto'):
|
||||
mail_session = GuerrillaMailSession()
|
||||
options['email'] = mail_session.get_session_state()['email_address']
|
||||
|
||||
|
||||
def get_driver():
|
||||
options = Options()
|
||||
options.add_argument('--headless')
|
||||
driver = webdriver.Firefox(
|
||||
options=options, service_log_path=os.devnull)
|
||||
driver.implicitly_wait(10)
|
||||
return driver
|
||||
|
||||
|
||||
def init_downloaded():
|
||||
with open(config.get('download_history_file'), 'r') as f:
|
||||
for line in f:
|
||||
downloaded.add(line.strip())
|
||||
|
||||
|
||||
def download_file(driver, album_data=None):
|
||||
logger.info('On download page')
|
||||
page_url = driver.find_element("xpath",
|
||||
xpath['album-link']).get_attribute('href')
|
||||
page_url = 'https://' + BASE_URL + urlsplit(page_url).path
|
||||
if album_data is None:
|
||||
album_data = mail_album_data[page_url]
|
||||
driver.find_element("xpath", xpath['formats']).click()
|
||||
wait()
|
||||
driver.find_element("xpath",
|
||||
f'//option[@value="{formats[options["format"]]}"]').click()
|
||||
logger.info(f'Set format to {formats[options["format"]]}')
|
||||
wait()
|
||||
button = driver.find_element("xpath", xpath['download'])
|
||||
WebDriverWait(driver, 60).until(EC.visibility_of(button))
|
||||
url = button.get_attribute('href')
|
||||
response = urllib.request.urlopen(url)
|
||||
name = os.path.basename(
|
||||
response.headers.get_filename().encode('latin-1').decode('utf-8'))
|
||||
length = int(response.getheader('content-length'))
|
||||
block_size = length // 10
|
||||
file_name = os.path.join(options['dir'], name)
|
||||
with open(file_name, 'wb') as f:
|
||||
size = 0
|
||||
while True:
|
||||
buf = response.read(block_size)
|
||||
if not buf:
|
||||
break
|
||||
f.write(buf)
|
||||
size += len(buf)
|
||||
logger.info(f'Downloading {name}, {size / length * 100: .1f}%')
|
||||
response.close()
|
||||
logger.info(f'Downloaded {file_name}')
|
||||
if file_name.endswith('zip') and not arguments['--no-unzip']:
|
||||
# Unzip archive
|
||||
dir_name = file_name[:-4]
|
||||
with zipfile.ZipFile(file_name, 'r') as f:
|
||||
f.extractall(dir_name)
|
||||
logger.info(f'Unzipped to {dir_name}. Use --no-unzip to prevent this')
|
||||
os.remove(file_name)
|
||||
files = glob.glob(os.path.join(dir_name, "*"))
|
||||
else:
|
||||
files = [file_name]
|
||||
# Tag downloaded audio files with url & comment
|
||||
logger.info("Setting tags...")
|
||||
for file in files:
|
||||
try:
|
||||
f = mutagen.File(file)
|
||||
if f is None:
|
||||
continue
|
||||
f['website'] = page_url
|
||||
if album_data['tags']:
|
||||
f['genre'] = album_data['tags']
|
||||
if album_data['about'] or album_data['credits']:
|
||||
comment = ''
|
||||
if album_data['about']:
|
||||
comment += album_data['about']
|
||||
if album_data['about'] and album_data['credits']:
|
||||
comment += '\n\n'
|
||||
if album_data['credits']:
|
||||
comment += album_data['credits']
|
||||
f['comment'] = comment
|
||||
f.save()
|
||||
except Exception as e:
|
||||
logger.info(f"Could not tag {file} - {e.__class__}: {e}")
|
||||
# successfully downloaded file, add to download history
|
||||
downloaded.add(page_url)
|
||||
with open(config.get('download_history_file'), 'a') as f:
|
||||
f.write(f'{page_url}\n')
|
||||
|
||||
|
||||
def download_files(driver, urls):
|
||||
for url in urls:
|
||||
driver.get(url)
|
||||
download_file(driver)
|
||||
|
||||
|
||||
def get_text(text):
|
||||
text = ''.join(line.strip() for line in text.split('\n'))
|
||||
text = text.replace('<br>', '\n')
|
||||
return BeautifulSoup(text, features='html.parser').get_text()
|
||||
|
||||
|
||||
def download_album(driver, url):
|
||||
# Remove url params
|
||||
url = urlsplit(url).geturl()
|
||||
if url in downloaded and not arguments['--force']:
|
||||
logger.error(
|
||||
f'{url} already downloaded. To download anyways, use option --force')
|
||||
return f'{url} already downloaded. To download anyways, use option --force'
|
||||
driver.get(url)
|
||||
wait()
|
||||
try:
|
||||
# Get album data
|
||||
album_data = {
|
||||
'about': None,
|
||||
'credits': None,
|
||||
'tags': None
|
||||
}
|
||||
try:
|
||||
s = driver.find_element("xpath",
|
||||
xpath['album-about']).get_attribute('innerHTML')
|
||||
s = get_text(s)
|
||||
album_data['about'] = s
|
||||
except Exception as e:
|
||||
logger.info(f"Could not get album about - {e.__class__}: {e}")
|
||||
try:
|
||||
s = driver.find_element("xpath",
|
||||
xpath['album-credits']).get_attribute('innerHTML')
|
||||
s = get_text(s)
|
||||
album_data['credits'] = s
|
||||
except Exception as e:
|
||||
logger.info(f"Could not get album credits - {e.__class__}: {e}")
|
||||
try:
|
||||
s = driver.find_element("xpath",
|
||||
xpath['album-tags']).get_attribute('innerHTML')
|
||||
tags = {a.text for a in BeautifulSoup(
|
||||
s, features='html.parser', parse_only=SoupStrainer('a'))}
|
||||
album_data['tags'] = ','.join(sorted(tags))
|
||||
except Exception as e:
|
||||
logger.info(f"Could not get album tags - {e.__class__}: {e}")
|
||||
|
||||
logger.info(f"Album data: {album_data}")
|
||||
|
||||
button = driver.find_element("xpath", xpath['buy'])
|
||||
if button.text == 'Free Download':
|
||||
logger.info(f'{url} is Free Download')
|
||||
button.click()
|
||||
wait()
|
||||
return download_file(driver, album_data)
|
||||
else:
|
||||
# name your price download
|
||||
logger.info(f'{url} is not Free Download')
|
||||
button.click()
|
||||
wait()
|
||||
price_input = driver.find_element("xpath", xpath['price'])
|
||||
price_input.click()
|
||||
price_input.send_keys('0')
|
||||
wait()
|
||||
try:
|
||||
driver.find_element("xpath", xpath['download-nyp']).click()
|
||||
except:
|
||||
logger.error(f'{url} is not free')
|
||||
return f'{url} is not free'
|
||||
checkout = driver.find_element("xpath", xpath['checkout'])
|
||||
if checkout.text == 'Download Now':
|
||||
checkout.click()
|
||||
wait()
|
||||
return download_file(driver, album_data)
|
||||
else:
|
||||
init_email()
|
||||
logger.info(f'{url} requires email')
|
||||
# fill out info
|
||||
driver.find_element("xpath",
|
||||
xpath['email']).send_keys(options['email'])
|
||||
wait()
|
||||
driver.find_element("xpath",
|
||||
xpath['zipcode']).send_keys(options['zipcode'])
|
||||
wait()
|
||||
Select(driver.find_element("xpath",
|
||||
xpath['country'])).select_by_visible_text(options['country'])
|
||||
wait()
|
||||
checkout.click()
|
||||
global expected_emails
|
||||
expected_emails += 1
|
||||
mail_album_data[url] = album_data
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading {url} - {e.__class__}: {e}")
|
||||
return e
|
||||
|
||||
|
||||
def download_albums(driver, urls):
|
||||
for link in urls:
|
||||
logger.info(f'Downloading {link}')
|
||||
download_album(driver, link)
|
||||
|
||||
|
||||
def download_label(driver, url):
|
||||
driver.get(url)
|
||||
global BASE_URL
|
||||
BASE_URL = urlsplit(url).netloc
|
||||
links = []
|
||||
for album in driver.find_elements("xpath", xpath['albums']):
|
||||
links.append(album.get_attribute('href'))
|
||||
download_albums(driver, links)
|
||||
|
||||
|
||||
def main():
|
||||
global arguments
|
||||
arguments = docopt(__doc__, version=__version__)
|
||||
if arguments['-a'] or arguments['-l']:
|
||||
# set options
|
||||
for option in options:
|
||||
arg = f'--{option}'
|
||||
if arguments[arg]:
|
||||
options[option] = arguments[arg][0]
|
||||
else:
|
||||
options[option] = config.get(option)
|
||||
if not options[option]:
|
||||
logger.error(
|
||||
f'{option} is not set, use "bcdl-free setdefault {arg} <{option}>"')
|
||||
sys.exit(1)
|
||||
if options['format'] not in formats:
|
||||
logger.error(
|
||||
f'{options["format"]} is not a valid format. See "bcdl-free -h" for valid formats')
|
||||
sys.exit(1)
|
||||
init_downloaded()
|
||||
driver = get_driver()
|
||||
try:
|
||||
if arguments['-a']:
|
||||
err = download_album(driver, arguments['-a'])
|
||||
if err:
|
||||
sys.exit(1)
|
||||
elif arguments['-l']:
|
||||
download_label(driver, arguments['-l'])
|
||||
elif arguments['setdefault']:
|
||||
# write arguments to config
|
||||
for option in options:
|
||||
arg = f'--{option}'
|
||||
if arguments[arg]:
|
||||
config.set(option, arguments[arg][0])
|
||||
elif arguments['defaults']:
|
||||
print(str(config))
|
||||
elif arguments['clear']:
|
||||
with open(config.get('download_history_file'), 'w'):
|
||||
pass
|
||||
# download emailed albums
|
||||
checked_ids = set()
|
||||
album_urls = set()
|
||||
global expected_emails
|
||||
logger.info(f'Waiting for {expected_emails} emails from bandcamp')
|
||||
while expected_emails > 0:
|
||||
time.sleep(10)
|
||||
try:
|
||||
for email in mail_session.get_email_list():
|
||||
if email.guid not in checked_ids:
|
||||
checked_ids.add(email.guid)
|
||||
if email.sender == 'noreply@bandcamp.com' and 'download' in email.subject:
|
||||
logger.info(f'Received email "{email.subject}"')
|
||||
email = mail_session.get_email(email.guid)
|
||||
match = link_regex.search(email.body)
|
||||
if match:
|
||||
download_url = match.group('url')
|
||||
album_urls.add(download_url)
|
||||
expected_emails -= 1
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
logger.info(f'Downloading {len(album_urls)} albums...')
|
||||
download_files(driver, album_urls)
|
||||
except:
|
||||
raise
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user