support collections
This commit is contained in:
parent
8c45dd23f1
commit
10e8a97b04
1 changed files with 71 additions and 19 deletions
|
|
@ -3,43 +3,83 @@
|
|||
from requests import Session
|
||||
from os import utime
|
||||
from os.path import exists, getmtime, getatime, getsize, abspath, join
|
||||
from sys import argv
|
||||
import argparse
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# args
|
||||
parser = argparse.ArgumentParser(description='Download items from steam workshop.')
|
||||
parser.add_argument('ids', metavar='ITEM_ID', nargs='+', type=int, help='steam workshop file ids')
|
||||
parser.add_argument('-o', '--out', metavar='TRAGET_DIR', dest='out', type=abspath, default='.', help='output dir')
|
||||
parser.add_argument('-i', '--items', metavar='ID', nargs='+', dest='item_ids', type=int, default=[], help='workshop item ids')
|
||||
parser.add_argument('-c', '--collections', metavar='ID', nargs='+', dest='collection_ids', type=int, default=[], help='workshop collection ids')
|
||||
parser.add_argument('-o', '--out', metavar='PATH', dest='out', type=abspath, default='.', help='output dir')
|
||||
parser.add_argument('-f', '--filename', metavar='PATTERN', dest='filename', type=str, default='{title}-{id}.{ext}', help='output file basename pattern: defaults to \'{title}-{id}.{ext}\', also supports {filename}')
|
||||
parser.add_argument('-p', '--parallel-downloads', metavar='NUMBER', dest='parallel_downloads', type=int, default=8, help='number of parallel downloads')
|
||||
args = parser.parse_args()
|
||||
print(f"item ids: {', '.join(str(id) for id in args.ids)}", flush=True)
|
||||
print(f"target dir: {args.out}", flush=True)
|
||||
|
||||
if not args.item_ids and not args.collection_ids:
|
||||
parser.error('at least one of --items or --collections must be specified')
|
||||
|
||||
# init http session
|
||||
session = Session()
|
||||
|
||||
# get item information
|
||||
def flash(string):
|
||||
print(f"\r\033[K{string}...", end="", flush=True)
|
||||
|
||||
def log(string):
|
||||
print(f"\r\033[K{string}")
|
||||
|
||||
# get items from collections
|
||||
if args.collection_ids:
|
||||
flash('resolving collections')
|
||||
response = session.post(
|
||||
'http://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1',
|
||||
'http://api.steampowered.com/ISteamRemoteStorage/GetCollectionDetails/v1',
|
||||
data={
|
||||
'itemcount' : len(args.ids),
|
||||
'collectioncount' : len(args.collection_ids),
|
||||
**{
|
||||
f'publishedfileids[{i}]' : plugin_id
|
||||
for i, plugin_id in enumerate(args.ids)
|
||||
f'publishedfileids[{i}]' : collection_id
|
||||
for i, collection_id in enumerate(args.collection_ids)
|
||||
},
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# download items
|
||||
for item in response.json()['response']['publishedfiledetails']:
|
||||
# extract item ids
|
||||
for collection in response.json()['response']['collectiondetails']:
|
||||
for child in collection.get('children', []):
|
||||
if child['filetype'] == 0:
|
||||
args.item_ids.append(int(child['publishedfileid']))
|
||||
|
||||
# get items
|
||||
flash('resolving items')
|
||||
response = session.post(
|
||||
'http://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1',
|
||||
data={
|
||||
'itemcount' : len(args.item_ids),
|
||||
**{
|
||||
f'publishedfileids[{i}]' : plugin_id
|
||||
for i, plugin_id in enumerate(args.item_ids)
|
||||
},
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# create thread pool
|
||||
executor = ThreadPoolExecutor(max_workers=args.parallel_downloads)
|
||||
|
||||
# download function
|
||||
def download(item):
|
||||
# file found?
|
||||
if item['result'] != 1:
|
||||
raise ValueError(f"getting file '{item['publishedfileid']}' info failed: {item}")
|
||||
|
||||
# get target path
|
||||
|
||||
target_path = join(args.out, f"{item['publishedfileid']}.vpk")
|
||||
print(f"- {item['title']}: ", end='', flush=True)
|
||||
filename_without_ext, ext = item['filename'].split('/')[-1].rsplit('.', 1)
|
||||
target_path = join(args.out, f"{args.filename}".format(
|
||||
id=item['publishedfileid'],
|
||||
title=re.sub(r'[^a-zA-Z0-9]+', '_', item['title']).strip('_'),
|
||||
filename=filename_without_ext,
|
||||
ext=ext,
|
||||
))
|
||||
|
||||
# skip item?
|
||||
if (
|
||||
|
|
@ -47,24 +87,36 @@ for item in response.json()['response']['publishedfiledetails']:
|
|||
int(item['time_updated']) == int(getmtime(target_path)) and # mtime matches
|
||||
int(item['file_size']) == int(getsize(target_path)) # filesize matches
|
||||
):
|
||||
print(f"already satisfied", flush=True)
|
||||
continue
|
||||
log(f'skipped {target_path}')
|
||||
return
|
||||
|
||||
# download item
|
||||
print(f"downloading", end='', flush=True)
|
||||
response = session.get(item['file_url'], stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
# one chunk at a time
|
||||
with open(target_path, 'wb') as file:
|
||||
for chunk in response.iter_content(chunk_size=65_536):
|
||||
print('.', end='', flush=True)
|
||||
written = 0
|
||||
for chunk in response.iter_content(chunk_size=1_048_576):
|
||||
if chunk:
|
||||
# print every 8MB
|
||||
written += len(chunk)
|
||||
if written // (8 * 1024 * 1024) > 0:
|
||||
flash(f'downloading {target_path}')
|
||||
written = 0
|
||||
|
||||
file.write(chunk)
|
||||
|
||||
# update modify time
|
||||
print(' done', flush=True)
|
||||
log(f'finished {target_path}')
|
||||
utime(target_path, (getatime(target_path), item['time_updated']))
|
||||
|
||||
# download items
|
||||
for item in response.json()['response']['publishedfiledetails']:
|
||||
executor.submit(download, item)
|
||||
|
||||
# wait for all downloads to finish
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
# close session
|
||||
session.close()
|
||||
|
|
|
|||
Loading…
Reference in a new issue