cronekorkn/spektors_thesaurus_scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

downlaod function

Browse source
This commit is contained in:
mwiegand 2023-04-21 19:21:00 +02:00
parent 9326ff73ca
commit 7045a713c8
No known key found for this signature in database
1 changed files with 13 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
scrape.py
View file

@ -5,6 +5,17 @@ from shutil import copyfileobj
from os.path import exists, splitext from os.path import exists, splitext
def download(url, filename):
if exists(filename):
print('exists:', filename)
return
else:
print('downloading:', filename)
with requests.get(url, stream=True) as stream:
with open(filename, 'wb') as file:
copyfileobj(stream.raw, file)
session = requests.Session() session = requests.Session()
response = session.get('https://www.spektorsthesaurus.com/songs') response = session.get('https://www.spektorsthesaurus.com/songs')
session_id = session.cookies.get_dict()['svSession'] session_id = session.cookies.get_dict()['svSession']
@ -99,15 +110,7 @@ for song in response.json()['items']:
_, extension = splitext(performance['bootleg']) _, extension = splitext(performance['bootleg'])
filename = f"{song['song_name']} - {performance['eventName']}{extension}".replace('/', '|') filename = f"{song['song_name']} - {performance['eventName']}{extension}".replace('/', '|')
if exists(filename): download(performance['bootleg'], filename)
print(filename, 'exists')
continue
else:
print('downloading', filename)
with requests.get(performance['bootleg'], stream=True) as stream:
with open(filename, 'wb') as file:
copyfileobj(stream.raw, file)
# demos # demos
response = session.post( response = session.post(
@ -155,14 +158,5 @@ for song in response.json()['items']:
_, extension = splitext(demo['link']) _, extension = splitext(demo['link'])
filename = f"{song['song_name']} - {demo['release_name']}{extension}".replace('/', '|') filename = f"{song['song_name']} - {demo['release_name']}{extension}".replace('/', '|')
if exists(filename): download(demo['link'], filename)
print(filename, 'exists')
continue
else:
print('downloading', filename)
with requests.get(demo['link'], stream=True) as stream:
with open(filename, 'wb') as file:
copyfileobj(stream.raw, file)