From 7045a713c84cd2e2cbfc9ccfa04dedad3aa8bf57 Mon Sep 17 00:00:00 2001 From: mwiegand Date: Fri, 21 Apr 2023 19:21:00 +0200 Subject: [PATCH] downlaod function --- scrape.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/scrape.py b/scrape.py index f075233..45dce8a 100755 --- a/scrape.py +++ b/scrape.py @@ -5,6 +5,17 @@ from shutil import copyfileobj from os.path import exists, splitext +def download(url, filename): + if exists(filename): + print('exists:', filename) + return + else: + print('downloading:', filename) + with requests.get(url, stream=True) as stream: + with open(filename, 'wb') as file: + copyfileobj(stream.raw, file) + + session = requests.Session() response = session.get('https://www.spektorsthesaurus.com/songs') session_id = session.cookies.get_dict()['svSession'] @@ -99,15 +110,7 @@ for song in response.json()['items']: _, extension = splitext(performance['bootleg']) filename = f"{song['song_name']} - {performance['eventName']}{extension}".replace('/', '|') - if exists(filename): - print(filename, 'exists') - continue - else: - print('downloading', filename) - - with requests.get(performance['bootleg'], stream=True) as stream: - with open(filename, 'wb') as file: - copyfileobj(stream.raw, file) + download(performance['bootleg'], filename) # demos response = session.post( @@ -155,14 +158,5 @@ for song in response.json()['items']: _, extension = splitext(demo['link']) filename = f"{song['song_name']} - {demo['release_name']}{extension}".replace('/', '|') - if exists(filename): - print(filename, 'exists') - continue - else: - print('downloading', filename) - - with requests.get(demo['link'], stream=True) as stream: - with open(filename, 'wb') as file: - copyfileobj(stream.raw, file) - + download(demo['link'], filename)