diff options
Diffstat (limited to 'jython-tosca-parser/src/main/resources/Lib/site-packages/pip/download.py')
| -rw-r--r-- | jython-tosca-parser/src/main/resources/Lib/site-packages/pip/download.py | 729 |
1 files changed, 0 insertions, 729 deletions
diff --git a/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/download.py b/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/download.py deleted file mode 100644 index 4c23f5e..0000000 --- a/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/download.py +++ /dev/null @@ -1,729 +0,0 @@ -import cgi -import email.utils -import hashlib -import getpass -import mimetypes -import os -import platform -import re -import shutil -import sys -import tempfile - -import pip - -from pip.backwardcompat import urllib, urlparse, raw_input -from pip.exceptions import InstallationError, HashMismatch -from pip.util import (splitext, rmtree, format_size, display_path, - backup_dir, ask_path_exists, unpack_file, - create_download_cache_folder, cache_download) -from pip.vcs import vcs -from pip.log import logger -from pip._vendor import requests, six -from pip._vendor.requests.adapters import BaseAdapter -from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth -from pip._vendor.requests.compat import IncompleteRead -from pip._vendor.requests.exceptions import InvalidURL, ChunkedEncodingError -from pip._vendor.requests.models import Response -from pip._vendor.requests.structures import CaseInsensitiveDict - -__all__ = ['get_file_content', - 'is_url', 'url_to_path', 'path_to_url', - 'is_archive_file', 'unpack_vcs_link', - 'unpack_file_url', 'is_vcs_url', 'is_file_url', 'unpack_http_url'] - - -def user_agent(): - """Return a string representing the user agent.""" - _implementation = platform.python_implementation() - - if _implementation == 'CPython': - _implementation_version = platform.python_version() - elif _implementation == 'PyPy': - _implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro) - if sys.pypy_version_info.releaselevel != 'final': - _implementation_version = ''.join([ - _implementation_version, - sys.pypy_version_info.releaselevel, - ]) - elif _implementation == 'Jython': - _implementation_version = platform.python_version() # Complete Guess - elif _implementation == 'IronPython': - _implementation_version = platform.python_version() # Complete Guess - else: - _implementation_version = 'Unknown' - - try: - p_system = platform.system() - p_release = platform.release() - except IOError: - p_system = 'Unknown' - p_release = 'Unknown' - - return " ".join(['pip/%s' % pip.__version__, - '%s/%s' % (_implementation, _implementation_version), - '%s/%s' % (p_system, p_release)]) - - -class MultiDomainBasicAuth(AuthBase): - - def __init__(self, prompting=True): - self.prompting = prompting - self.passwords = {} - - def __call__(self, req): - parsed = urlparse.urlparse(req.url) - - # Get the netloc without any embedded credentials - netloc = parsed.netloc.split("@", 1)[-1] - - # Set the url of the request to the url without any credentials - req.url = urlparse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) - - # Use any stored credentials that we have for this netloc - username, password = self.passwords.get(netloc, (None, None)) - - # Extract credentials embedded in the url if we have none stored - if username is None: - username, password = self.parse_credentials(parsed.netloc) - - if username or password: - # Store the username and password - self.passwords[netloc] = (username, password) - - # Send the basic auth with this request - req = HTTPBasicAuth(username or "", password or "")(req) - - # Attach a hook to handle 401 responses - req.register_hook("response", self.handle_401) - - return req - - def handle_401(self, resp, **kwargs): - # We only care about 401 responses, anything else we want to just - # pass through the actual response - if resp.status_code != 401: - return resp - - # We are not able to prompt the user so simple return the response - if not self.prompting: - return resp - - parsed = urlparse.urlparse(resp.url) - - # Prompt the user for a new username and password - username = raw_input("User for %s: " % parsed.netloc) - password = getpass.getpass("Password: ") - - # Store the new username and password to use for future requests - if username or password: - self.passwords[parsed.netloc] = (username, password) - - # Consume content and release the original connection to allow our new - # request to reuse the same one. - resp.content - resp.raw.release_conn() - - # Add our new username and password to the request - req = HTTPBasicAuth(username or "", password or "")(resp.request) - - # Send our new request - new_resp = resp.connection.send(req, **kwargs) - new_resp.history.append(resp) - - return new_resp - - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - return userinfo.split(":", 1) - return userinfo, None - return None, None - - -class LocalFSResponse(object): - - def __init__(self, fileobj): - self.fileobj = fileobj - - def __getattr__(self, name): - return getattr(self.fileobj, name) - - def read(self, amt=None, decode_content=None, cache_content=False): - return self.fileobj.read(amt) - - # Insert Hacks to Make Cookie Jar work w/ Requests - @property - def _original_response(self): - class FakeMessage(object): - def getheaders(self, header): - return [] - - def get_all(self, header, default): - return [] - - class FakeResponse(object): - @property - def msg(self): - return FakeMessage() - - return FakeResponse() - - -class LocalFSAdapter(BaseAdapter): - - def send(self, request, stream=None, timeout=None, verify=None, cert=None, - proxies=None): - parsed_url = urlparse.urlparse(request.url) - - # We only work for requests with a host of localhost - if parsed_url.netloc.lower() != "localhost": - raise InvalidURL( - "Invalid URL %r: Only localhost is allowed" % - request.url - ) - - real_url = urlparse.urlunparse(parsed_url[:1] + ("",) + parsed_url[2:]) - pathname = url_to_path(real_url) - - resp = Response() - resp.status_code = 200 - resp.url = real_url - - stats = os.stat(pathname) - modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - resp.headers = CaseInsensitiveDict({ - "Content-Type": mimetypes.guess_type(pathname)[0] or "text/plain", - "Content-Length": stats.st_size, - "Last-Modified": modified, - }) - - resp.raw = LocalFSResponse(open(pathname, "rb")) - resp.close = resp.raw.close - - return resp - - def close(self): - pass - - -class PipSession(requests.Session): - - timeout = None - - def __init__(self, *args, **kwargs): - retries = kwargs.pop('retries', None) - - super(PipSession, self).__init__(*args, **kwargs) - - # Attach our User Agent to the request - self.headers["User-Agent"] = user_agent() - - # Attach our Authentication handler to the session - self.auth = MultiDomainBasicAuth() - - # Configure retries - if retries: - http_adapter = requests.adapters.HTTPAdapter(max_retries=retries) - self.mount("http://", http_adapter) - self.mount("https://", http_adapter) - - # Enable file:// urls - self.mount("file://", LocalFSAdapter()) - - def request(self, method, url, *args, **kwargs): - # Make file:// urls not fail due to lack of a hostname - parsed = urlparse.urlparse(url) - if parsed.scheme == "file": - url = urlparse.urlunparse(parsed[:1] + ("localhost",) + parsed[2:]) - - # Allow setting a default timeout on a session - kwargs.setdefault("timeout", self.timeout) - - # Dispatch the actual request - return super(PipSession, self).request(method, url, *args, **kwargs) - - -def get_file_content(url, comes_from=None, session=None): - """Gets the content of a file; it may be a filename, file: URL, or - http: URL. Returns (location, content). Content is unicode.""" - if session is None: - session = PipSession() - - match = _scheme_re.search(url) - if match: - scheme = match.group(1).lower() - if (scheme == 'file' and comes_from - and comes_from.startswith('http')): - raise InstallationError( - 'Requirements file %s references URL %s, which is local' - % (comes_from, url)) - if scheme == 'file': - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path - else: - # FIXME: catch some errors - resp = session.get(url) - resp.raise_for_status() - - if six.PY3: - return resp.url, resp.text - else: - return resp.url, resp.content - try: - f = open(url) - content = f.read() - except IOError as exc: - raise InstallationError( - 'Could not open requirements file: %s' % str(exc) - ) - else: - f.close() - return url, content - - -_scheme_re = re.compile(r'^(http|https|file):', re.I) -_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) - - -def is_url(name): - """Returns true if the name looks like a URL""" - if ':' not in name: - return False - scheme = name.split(':', 1)[0].lower() - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - path = url[len('file:'):].lstrip('/') - path = urllib.unquote(path) - if _url_drive_re.match(path): - path = path[0] + ':' + path[2:] - else: - path = '/' + path - return path - - -_drive_re = re.compile('^([a-z]):', re.I) -_url_drive_re = re.compile('^([a-z])[:|]', re.I) - - -def path_to_url(path): - """ - Convert a path to a file: URL. The path will be made absolute and have - quoted path parts. - """ - path = os.path.normpath(os.path.abspath(path)) - drive, path = os.path.splitdrive(path) - filepath = path.split(os.path.sep) - url = '/'.join([urllib.quote(part) for part in filepath]) - if not drive: - url = url.lstrip('/') - return 'file:///' + drive + url - - -def is_archive_file(name): - """Return True if `name` is a considered as an archive file.""" - archives = ( - '.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle', '.whl' - ) - ext = splitext(name)[1].lower() - if ext in archives: - return True - return False - - -def unpack_vcs_link(link, location, only_download=False): - vcs_backend = _get_used_vcs_backend(link) - if only_download: - vcs_backend.export(location) - else: - vcs_backend.unpack(location) - - -def _get_used_vcs_backend(link): - for backend in vcs.backends: - if link.scheme in backend.schemes: - vcs_backend = backend(link.url) - return vcs_backend - - -def is_vcs_url(link): - return bool(_get_used_vcs_backend(link)) - - -def is_file_url(link): - return link.url.lower().startswith('file:') - - -def _check_hash(download_hash, link): - if download_hash.digest_size != hashlib.new(link.hash_name).digest_size: - logger.fatal( - "Hash digest size of the package %d (%s) doesn't match the " - "expected hash name %s!" % - (download_hash.digest_size, link, link.hash_name) - ) - raise HashMismatch('Hash name mismatch for package %s' % link) - if download_hash.hexdigest() != link.hash: - logger.fatal( - "Hash of the package %s (%s) doesn't match the expected hash %s!" % - (link, download_hash.hexdigest(), link.hash) - ) - raise HashMismatch( - 'Bad %s hash for package %s' % (link.hash_name, link) - ) - - -def _get_hash_from_file(target_file, link): - try: - download_hash = hashlib.new(link.hash_name) - except (ValueError, TypeError): - logger.warn( - "Unsupported hash name %s for package %s" % (link.hash_name, link) - ) - return None - - fp = open(target_file, 'rb') - while True: - chunk = fp.read(4096) - if not chunk: - break - download_hash.update(chunk) - fp.close() - return download_hash - - -def _download_url(resp, link, temp_location): - fp = open(temp_location, 'wb') - download_hash = None - if link.hash and link.hash_name: - try: - download_hash = hashlib.new(link.hash_name) - except ValueError: - logger.warn( - "Unsupported hash name %s for package %s" % - (link.hash_name, link) - ) - try: - total_length = int(resp.headers['content-length']) - except (ValueError, KeyError, TypeError): - total_length = 0 - downloaded = 0 - show_progress = total_length > 40 * 1000 or not total_length - show_url = link.show_url - try: - if show_progress: - # FIXME: the URL can get really long in this message: - if total_length: - logger.start_progress( - 'Downloading %s (%s): ' % - (show_url, format_size(total_length)) - ) - else: - logger.start_progress( - 'Downloading %s (unknown size): ' % show_url - ) - else: - logger.notify('Downloading %s' % show_url) - logger.info('Downloading from URL %s' % link) - - def resp_read(chunk_size): - try: - # Special case for urllib3. - try: - for chunk in resp.raw.stream( - chunk_size, - # We use decode_content=False here because we do - # want urllib3 to mess with the raw bytes we get - # from the server. If we decompress inside of - # urllib3 then we cannot verify the checksum - # because the checksum will be of the compressed - # file. This breakage will only occur if the - # server adds a Content-Encoding header, which - # depends on how the server was configured: - # - Some servers will notice that the file isn't a - # compressible file and will leave the file alone - # and with an empty Content-Encoding - # - Some servers will notice that the file is - # already compressed and will leave the file - # alone and will add a Content-Encoding: gzip - # header - # - Some servers won't notice anything at all and - # will take a file that's already been compressed - # and compress it again and set the - # Content-Encoding: gzip header - # - # By setting this not to decode automatically we - # hope to eliminate problems with the second case. - decode_content=False): - yield chunk - except IncompleteRead as e: - raise ChunkedEncodingError(e) - except AttributeError: - # Standard file-like object. - while True: - chunk = resp.raw.read(chunk_size) - if not chunk: - break - yield chunk - - for chunk in resp_read(4096): - downloaded += len(chunk) - if show_progress: - if not total_length: - logger.show_progress('%s' % format_size(downloaded)) - else: - logger.show_progress( - '%3i%% %s' % - ( - 100 * downloaded / total_length, - format_size(downloaded) - ) - ) - if download_hash is not None: - download_hash.update(chunk) - fp.write(chunk) - fp.close() - finally: - if show_progress: - logger.end_progress('%s downloaded' % format_size(downloaded)) - return download_hash - - -def _copy_file(filename, location, content_type, link): - copy = True - download_location = os.path.join(location, link.filename) - if os.path.exists(download_location): - response = ask_path_exists( - 'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' % - display_path(download_location), ('i', 'w', 'b')) - if response == 'i': - copy = False - elif response == 'w': - logger.warn('Deleting %s' % display_path(download_location)) - os.remove(download_location) - elif response == 'b': - dest_file = backup_dir(download_location) - logger.warn( - 'Backing up %s to %s' % - (display_path(download_location), display_path(dest_file)) - ) - shutil.move(download_location, dest_file) - if copy: - shutil.copy(filename, download_location) - logger.notify('Saved %s' % display_path(download_location)) - - -def unpack_http_url(link, location, download_cache, download_dir=None, - session=None): - if session is None: - session = PipSession() - - temp_dir = tempfile.mkdtemp('-unpack', 'pip-') - temp_location = None - target_url = link.url.split('#', 1)[0] - already_cached = False - cache_file = None - cache_content_type_file = None - download_hash = None - - # If a download cache is specified, is the file cached there? - if download_cache: - cache_file = os.path.join( - download_cache, - urllib.quote(target_url, '') - ) - cache_content_type_file = cache_file + '.content-type' - already_cached = ( - os.path.exists(cache_file) and - os.path.exists(cache_content_type_file) - ) - if not os.path.isdir(download_cache): - create_download_cache_folder(download_cache) - - # If a download dir is specified, is the file already downloaded there? - already_downloaded = None - if download_dir: - already_downloaded = os.path.join(download_dir, link.filename) - if not os.path.exists(already_downloaded): - already_downloaded = None - - # If already downloaded, does its hash match? - if already_downloaded: - temp_location = already_downloaded - content_type = mimetypes.guess_type(already_downloaded)[0] - logger.notify('File was already downloaded %s' % already_downloaded) - if link.hash: - download_hash = _get_hash_from_file(temp_location, link) - try: - _check_hash(download_hash, link) - except HashMismatch: - logger.warn( - 'Previously-downloaded file %s has bad hash, ' - 're-downloading.' % temp_location - ) - temp_location = None - os.unlink(already_downloaded) - already_downloaded = None - - # If not a valid download, let's confirm the cached file is valid - if already_cached and not temp_location: - with open(cache_content_type_file) as fp: - content_type = fp.read().strip() - temp_location = cache_file - logger.notify('Using download cache from %s' % cache_file) - if link.hash and link.hash_name: - download_hash = _get_hash_from_file(cache_file, link) - try: - _check_hash(download_hash, link) - except HashMismatch: - logger.warn( - 'Cached file %s has bad hash, ' - 're-downloading.' % temp_location - ) - temp_location = None - os.unlink(cache_file) - os.unlink(cache_content_type_file) - already_cached = False - - # We don't have either a cached or a downloaded copy - # let's download to a tmp dir - if not temp_location: - try: - resp = session.get( - target_url, - # We use Accept-Encoding: identity here because requests - # defaults to accepting compressed responses. This breaks in - # a variety of ways depending on how the server is configured. - # - Some servers will notice that the file isn't a compressible - # file and will leave the file alone and with an empty - # Content-Encoding - # - Some servers will notice that the file is already - # compressed and will leave the file alone and will add a - # Content-Encoding: gzip header - # - Some servers won't notice anything at all and will take - # a file that's already been compressed and compress it again - # and set the Content-Encoding: gzip header - # By setting this to request only the identity encoding We're - # hoping to eliminate the third case. Hopefully there does not - # exist a server which when given a file will notice it is - # already compressed and that you're not asking for a - # compressed file and will then decompress it before sending - # because if that's the case I don't think it'll ever be - # possible to make this work. - headers={"Accept-Encoding": "identity"}, - stream=True, - ) - resp.raise_for_status() - except requests.HTTPError as exc: - logger.fatal("HTTP error %s while getting %s" % - (exc.response.status_code, link)) - raise - - content_type = resp.headers.get('content-type', '') - filename = link.filename # fallback - # Have a look at the Content-Disposition header for a better guess - content_disposition = resp.headers.get('content-disposition') - if content_disposition: - type, params = cgi.parse_header(content_disposition) - # We use ``or`` here because we don't want to use an "empty" value - # from the filename param. - filename = params.get('filename') or filename - ext = splitext(filename)[1] - if not ext: - ext = mimetypes.guess_extension(content_type) - if ext: - filename += ext - if not ext and link.url != resp.url: - ext = os.path.splitext(resp.url)[1] - if ext: - filename += ext - temp_location = os.path.join(temp_dir, filename) - download_hash = _download_url(resp, link, temp_location) - if link.hash and link.hash_name: - _check_hash(download_hash, link) - - # a download dir is specified; let's copy the archive there - if download_dir and not already_downloaded: - _copy_file(temp_location, download_dir, content_type, link) - - # unpack the archive to the build dir location. even when only downloading - # archives, they have to be unpacked to parse dependencies - unpack_file(temp_location, location, content_type, link) - - # if using a download cache, cache it, if needed - if cache_file and not already_cached: - cache_download(cache_file, temp_location, content_type) - - if not (already_cached or already_downloaded): - os.unlink(temp_location) - - os.rmdir(temp_dir) - - -def unpack_file_url(link, location, download_dir=None): - - link_path = url_to_path(link.url_without_fragment) - already_downloaded = False - - # If it's a url to a local directory - if os.path.isdir(link_path): - if os.path.isdir(location): - rmtree(location) - shutil.copytree(link_path, location, symlinks=True) - return - - # if link has a hash, let's confirm it matches - if link.hash: - link_path_hash = _get_hash_from_file(link_path, link) - _check_hash(link_path_hash, link) - - # If a download dir is specified, is the file already there and valid? - if download_dir: - download_path = os.path.join(download_dir, link.filename) - if os.path.exists(download_path): - content_type = mimetypes.guess_type(download_path)[0] - logger.notify('File was already downloaded %s' % download_path) - if link.hash: - download_hash = _get_hash_from_file(download_path, link) - try: - _check_hash(download_hash, link) - already_downloaded = True - except HashMismatch: - logger.warn( - 'Previously-downloaded file %s has bad hash, ' - 're-downloading.' % link_path - ) - os.unlink(download_path) - else: - already_downloaded = True - - if already_downloaded: - from_path = download_path - else: - from_path = link_path - - content_type = mimetypes.guess_type(from_path)[0] - - # unpack the archive to the build dir location. even when only downloading - # archives, they have to be unpacked to parse dependencies - unpack_file(from_path, location, content_type, link) - - # a download dir is specified and not already downloaded - if download_dir and not already_downloaded: - _copy_file(from_path, download_dir, content_type, link) |