blob: c8357e91cf7d05885c46abb37dda53c6970c015d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
HTTP utilities.
"""
import os
import tempfile
import requests
def download_file(url, destination=None, logger=None, progress_handler=None):
"""
Download file.
:param url: URL from which to download
:type url: basestring
:param destination: path where the file should be saved or ``None`` to auto-generate
:type destination: basestring
:returns: path where the file was saved
:rtype: basestring
:raises exceptions.IOError:
:raises requests.exceptions.RequestException:
"""
chunk_size = 1024
if not destination:
file_descriptor, destination = tempfile.mkstemp()
os.close(file_descriptor)
if logger:
logger.info('Downloading {0} to {1}...'.format(url, destination))
response = requests.get(url, stream=True)
final_url = response.url
if final_url != url and logger:
logger.debug('Redirected to {0}'.format(final_url))
read_bytes = 0
total_size = int(response.headers['Content-Length']) \
if 'Content-Length' in response.headers else None
try:
with open(destination, 'wb') as destination_file:
for chunk in response.iter_content(chunk_size):
destination_file.write(chunk)
if total_size and progress_handler:
# Only showing progress bar if we have the total content length
read_bytes += chunk_size
progress_handler(read_bytes, total_size)
finally:
response.close()
return destination
|
