Last active
May 9, 2022 20:53
-
-
Save tgsmith61591/353fd3a852e83697a51c4cd5951a6ca6 to your computer and use it in GitHub Desktop.
Download a file from the web using requests and a pretty progress bar
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| # | |
| # Download a file from the web using requests with a nice progress bar. | |
| from __future__ import print_function | |
| from tqdm import tqdm | |
| import requests | |
| import warnings | |
| import argparse | |
| import math | |
| def download(url, target, headers, block_size): | |
| # Stream so we can iterate the response. | |
| r = requests.get(url, stream=True, headers=headers) | |
| # Total size in bytes. | |
| total_size = int(r.headers.get('content-length', 0)) | |
| bytes_written = 0 | |
| with open(target, 'wb') as f: | |
| for data in tqdm(r.iter_content(block_size), | |
| total=math.ceil(total_size // block_size), | |
| unit='KB', unit_scale=True): | |
| bytes_written += len(data) | |
| f.write(data) | |
| if total_size != 0 and bytes_written != total_size: | |
| warnings.warn("Only read %i/%i bytes!" % (bytes_written, total_size)) | |
| elif total_size == 0: | |
| warnings.warn("Downloaded 0 bytes!") | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description="Download a file over HTTP " | |
| "with a nice progress bar.") | |
| parser.add_argument("--url", dest="url", type=str, | |
| help="The URL from which to download.") | |
| parser.add_argument("--output", dest="output", type=str, | |
| help="The absolute path to the file to which to save " | |
| "the web content.") | |
| parser.add_argument("--headers", dest="headers", type=str, default=None, | |
| help="Optional headers.") | |
| parser.add_argument("--block-size", dest="block_size", type=int, | |
| default=1024, help="The (optional) block size.") | |
| parser.set_defaults(url=None, output=None, headers=None, block_size=1024) | |
| args = parser.parse_args() | |
| # validate args | |
| if not all(arg for arg in (args.url, args.output)): | |
| raise ValueError("URL and output required!") | |
| download(url=args.url, target=args.output, headers=args.headers, | |
| block_size=args.block_size) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment