import os import shutil import zipfile from pip._vendor import pkg_resources from pip.backwardcompat import HTTPError from pip.download import (PipSession, url_to_path, unpack_vcs_link, is_vcs_url, is_file_url, unpack_file_url, unpack_http_url) from pip.exceptions import (InstallationError, BestVersionAlreadyInstalled, DistributionNotFound, PreviousBuildDirError) from pip.index import Link from pip.locations import (PIP_DELETE_MARKER_FILENAME, build_prefix, write_delete_marker_file) from pip.log import logger from pip.req.req_install import InstallRequirement from pip.util import (display_path, rmtree, dist_in_usersite, call_subprocess, _make_build_dir) from pip.vcs import vcs from pip.wheel import wheel_ext class Requirements(object): def __init__(self): self._keys = [] self._dict = {} def keys(self): return self._keys def values(self): return [self._dict[key] for key in self._keys] def __contains__(self, item): return item in self._keys def __setitem__(self, key, value): if key not in self._keys: self._keys.append(key) self._dict[key] = value def __getitem__(self, key): return self._dict[key] def __repr__(self): values = ['%s: %s' % (repr(k), repr(self[k])) for k in self.keys()] return 'Requirements({%s})' % ', '.join(values) class RequirementSet(object): def __init__(self, build_dir, src_dir, download_dir, download_cache=None, upgrade=False, ignore_installed=False, as_egg=False, target_dir=None, ignore_dependencies=False, force_reinstall=False, use_user_site=False, session=None, pycompile=True, wheel_download_dir=None): self.build_dir = build_dir self.src_dir = src_dir self.download_dir = download_dir if download_cache: download_cache = os.path.expanduser(download_cache) self.download_cache = download_cache self.upgrade = upgrade self.ignore_installed = ignore_installed self.force_reinstall = force_reinstall self.requirements = Requirements() # Mapping of alias: real_name self.requirement_aliases = {} self.unnamed_requirements = [] self.ignore_dependencies = ignore_dependencies self.successfully_downloaded = [] self.successfully_installed = [] self.reqs_to_cleanup = [] self.as_egg = as_egg self.use_user_site = use_user_site self.target_dir = target_dir # set from --target option self.session = session or PipSession() self.pycompile = pycompile self.wheel_download_dir = wheel_download_dir def __str__(self): reqs = [req for req in self.requirements.values() if not req.comes_from] reqs.sort(key=lambda req: req.name.lower()) return ' '.join([str(req.req) for req in reqs]) def add_requirement(self, install_req): name = install_req.name install_req.as_egg = self.as_egg install_req.use_user_site = self.use_user_site install_req.target_dir = self.target_dir install_req.pycompile = self.pycompile if not name: # url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) else: if self.has_requirement(name): raise InstallationError( 'Double requirement given: %s (already in %s, name=%r)' % (install_req, self.get_requirement(name), name)) self.requirements[name] = install_req # FIXME: what about other normalizations? E.g., _ vs. -? if name.lower() != name: self.requirement_aliases[name.lower()] = name def has_requirement(self, project_name): for name in project_name, project_name.lower(): if name in self.requirements or name in self.requirement_aliases: return True return False @property def has_requirements(self): return list(self.requirements.values()) or self.unnamed_requirements @property def has_editables(self): if any(req.editable for req in self.requirements.values()): return True if any(req.editable for req in self.unnamed_requirements): return True return False @property def is_download(self): if self.download_dir: self.download_dir = os.path.expanduser(self.download_dir) if os.path.exists(self.download_dir): return True else: logger.fatal('Could not find download directory') raise InstallationError( "Could not find or access download directory '%s'" % display_path(self.download_dir)) return False def get_requirement(self, project_name): for name in project_name, project_name.lower(): if name in self.requirements: return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] raise KeyError("No project with the name %r" % project_name) def uninstall(self, auto_confirm=False): for req in self.requirements.values(): req.uninstall(auto_confirm=auto_confirm) req.commit_uninstall() def locate_files(self): # FIXME: duplicates code from prepare_files; relevant code should # probably be factored out into a separate method unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install_needed = True if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: # don't uninstall conflict if user install and and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install_needed = False if req_to_install.satisfied_by: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: if req_to_install.source_dir is None: req_to_install.source_dir = req_to_install.build_location( self.src_dir ) elif install_needed: req_to_install.source_dir = req_to_install.build_location( self.build_dir, not self.is_download, ) if (req_to_install.source_dir is not None and not os.path.isdir(req_to_install.source_dir)): raise InstallationError( 'Could not install requirement %s because source folder %s' ' does not exist (perhaps --no-download was used without ' 'first running an equivalent install with --no-install?)' % (req_to_install, req_to_install.source_dir) ) def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """ Prepare process. Create temp directories, download and/or unpack files. """ unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None # ############################################# # # # Search for archive to fulfill requirement # # # ############################################# # if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound as exc: not_found = exc else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if (req_to_install.url and req_to_install.url.lower().startswith('file:')): logger.notify( 'Unpacking %s' % display_path(url_to_path(req_to_install.url)) ) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 # ################################ # # # vcs update or unpack archive # # # ################################ # try: is_bundle = False is_wheel = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory # NB: This call can result in the creation of a temporary # build directory location = req_to_install.build_location( self.build_dir, not self.is_download, ) unpack = True url = None # In the case where the req comes from a bundle, we should # assume a build dir exists and move on if req_to_install.from_bundle: pass # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. elif os.path.exists(os.path.join(location, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing buld directory (%s). This is likely" " due to a previous installation that failed. pip " "is being responsible and not assuming it can " "delete this. Please delete it and try again." % (req_to_install, location) ) else: # FIXME: this won't upgrade when there's an existing # package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement( req_to_install, upgrade=self.upgrade, ) else: # FIXME: should req_to_install.url already be a # link? url = Link(req_to_install.url) assert url if url: try: if ( url.filename.endswith(wheel_ext) and self.wheel_download_dir ): # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True else: download_dir = self.download_dir do_download = self.is_download self.unpack_url( url, location, download_dir, do_download, ) except HTTPError as exc: logger.fatal( 'Could not install requirement %s because ' 'of error %s' % (req_to_install, exc) ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, url) ) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle is_wheel = url and url.filename.endswith(wheel_ext) if is_bundle: req_to_install.move_bundle_files( self.build_dir, self.src_dir, ) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location if not is_wheel: # FIXME:https://github.com/pypa/pip/issues/1112 req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) elif is_wheel: req_to_install.source_dir = location req_to_install.url = url.url else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the # .egg-info/ directory is created for packing # in the bundle req_to_install.run_egg_info( force_root_egg_info=True, ) req_to_install.assert_source_matches_version() # @@ sketchy way of identifying packages not # grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.notify( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s' % req_to_install ) install = False # ###################### # # # parse dependencies # # # ###################### # if is_wheel: dist = list( pkg_resources.find_distributions(location) )[0] if not req_to_install.req: req_to_install.req = dist.as_requirement() self.add_requirement(req_to_install) if not self.ignore_dependencies: for subreq in dist.requires( req_to_install.extras): if self.has_requirement( subreq.project_name): continue subreq = InstallRequirement(str(subreq), req_to_install) reqs.append(subreq) self.add_requirement(subreq) # sdists elif not is_bundle: if (req_to_install.extras): logger.notify( "Installing extra requirements: %r" % ','.join(req_to_install.extras) ) if not self.ignore_dependencies: for req in req_to_install.requirements( req_to_install.extras): try: name = pkg_resources.Requirement.parse( req ).project_name except ValueError as exc: # FIXME: proper warning logger.error( 'Invalid requirement: %r (%s) in ' 'requirement %s' % (req, exc, req_to_install) ) continue if self.has_requirement(name): # FIXME: check for conflict continue subreq = InstallRequirement(req, req_to_install) reqs.append(subreq) self.add_requirement(subreq) if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install) # cleanup tmp src if not is_bundle: if ( self.is_download or req_to_install._temp_build_dir is not None ): self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if (bundle and ( req_to_install.url and req_to_install.url.startswith('file:///') )): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2 def cleanup_files(self, bundle=False): """Clean up files, remove builds.""" logger.notify('Cleaning up...') logger.indent += 2 for req in self.reqs_to_cleanup: req.remove_temporary_source() remove_dir = [] if self._pip_has_created_build_dir(): remove_dir.append(self.build_dir) # The source dir of a bundle can always be removed. # FIXME: not if it pre-existed the bundle! if bundle: remove_dir.append(self.src_dir) for dir in remove_dir: if os.path.exists(dir): logger.info('Removing temporary dir %s...' % dir) rmtree(dir) logger.indent -= 2 def _pip_has_created_build_dir(self): return ( self.build_dir == build_prefix and os.path.exists( os.path.join(self.build_dir, PIP_DELETE_MARKER_FILENAME) ) ) def copy_to_build_dir(self, req_to_install): target_dir = req_to_install.editable and self.src_dir or self.build_dir logger.info("Copying %s to %s" % (req_to_install.name, target_dir)) dest = os.path.join(target_dir, req_to_install.name) shutil.copytree(req_to_install.source_dir, dest) call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest, command_desc='python setup.py clean') def unpack_url(self, link, location, download_dir=None, only_download=False): if download_dir is None: download_dir = self.download_dir # non-editable vcs urls if is_vcs_url(link): if only_download: loc = download_dir else: loc = location unpack_vcs_link(link, loc, only_download) # file urls elif is_file_url(link): unpack_file_url(link, location, download_dir) if only_download: write_delete_marker_file(location) # http urls else: unpack_http_url( link, location, self.download_cache, download_dir, self.session, ) if only_download: write_delete_marker_file(location) def install(self, install_options, global_options=(), *args, **kwargs): """ Install everything in this set (after having downloaded and unpacked the packages) """ to_install = [r for r in self.requirements.values() if not r.satisfied_by] # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts) # move the distribute-0.7.X wrapper to the end because it does not # install a setuptools package. by moving it to the end, we ensure it's # setuptools dependency is handled first, which will provide the # setuptools package # TODO: take this out later distribute_req = pkg_resources.Requirement.parse("distribute>=0.7") for req in to_install: if (req.name == 'distribute' and req.installed_version in distribute_req): to_install.remove(req) to_install.append(req) if to_install: logger.notify( 'Installing collected packages: %s' % ', '.join([req.name for req in to_install]) ) logger.indent += 2 try: for requirement in to_install: # DISTRIBUTE TO SETUPTOOLS UPGRADE HACK (1 of 3 parts) # when upgrading from distribute-0.6.X to the new merged # setuptools in py2, we need to force setuptools to uninstall # distribute. In py3, which is always using distribute, this # conversion is already happening in distribute's # pkg_resources. It's ok *not* to check if setuptools>=0.7 # because if someone were actually trying to ugrade from # distribute to setuptools 0.6.X, then all this could do is # actually help, although that upgade path was certainly never # "supported" # TODO: remove this later if requirement.name == 'setuptools': try: # only uninstall distribute<0.7. For >=0.7, setuptools # will also be present, and that's what we need to # uninstall distribute_requirement = \ pkg_resources.Requirement.parse("distribute<0.7") existing_distribute = \ pkg_resources.get_distribution("distribute") if existing_distribute in distribute_requirement: requirement.conflicts_with = existing_distribute except pkg_resources.DistributionNotFound: # distribute wasn't installed, so nothing to do pass if requirement.conflicts_with: logger.notify('Found existing installation: %s' % requirement.conflicts_with) logger.indent += 2 try: requirement.uninstall(auto_confirm=True) finally: logger.indent -= 2 try: requirement.install( install_options, global_options, *args, **kwargs ) except: # if install did not succeed, rollback previous uninstall if (requirement.conflicts_with and not requirement.install_succeeded): requirement.rollback_uninstall() raise else: if (requirement.conflicts_with and requirement.install_succeeded): requirement.commit_uninstall() requirement.remove_temporary_source() finally: logger.indent -= 2 self.successfully_installed = to_install def create_bundle(self, bundle_filename): # FIXME: can't decide which is better; zip is easier to read # random files from, but tar.bz2 is smaller and not as lame a # format. # FIXME: this file should really include a manifest of the # packages, maybe some other metadata files. It would make # it easier to detect as well. zip = zipfile.ZipFile(bundle_filename, 'w', zipfile.ZIP_DEFLATED) vcs_dirs = [] for dir, basename in (self.build_dir, 'build'), (self.src_dir, 'src'): dir = os.path.normcase(os.path.abspath(dir)) for dirpath, dirnames, filenames in os.walk(dir): for backend in vcs.backends: vcs_backend = backend() vcs_url = vcs_rev = None if vcs_backend.dirname in dirnames: for vcs_dir in vcs_dirs: if dirpath.startswith(vcs_dir): # vcs bundle file already in parent directory break else: vcs_url, vcs_rev = vcs_backend.get_info( os.path.join(dir, dirpath)) vcs_dirs.append(dirpath) vcs_bundle_file = vcs_backend.bundle_file vcs_guide = vcs_backend.guide % {'url': vcs_url, 'rev': vcs_rev} dirnames.remove(vcs_backend.dirname) break if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: dirname = os.path.join(dirpath, dirname) name = self._clean_zip_name(dirname, dir) zip.writestr(basename + '/' + name + '/', '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue filename = os.path.join(dirpath, filename) name = self._clean_zip_name(filename, dir) zip.write(filename, basename + '/' + name) if vcs_url: name = os.path.join(dirpath, vcs_bundle_file) name = self._clean_zip_name(name, dir) zip.writestr(basename + '/' + name, vcs_guide) zip.writestr('pip-manifest.txt', self.bundle_requirements()) zip.close() BUNDLE_HEADER = '''\ # This is a pip bundle file, that contains many source packages # that can be installed as a group. You can install this like: # pip this_file.zip # The rest of the file contains a list of all the packages included: ''' def bundle_requirements(self): parts = [self.BUNDLE_HEADER] for req in [req for req in self.requirements.values() if not req.comes_from]: parts.append('%s==%s\n' % (req.name, req.installed_version)) parts.append( '# These packages were installed to satisfy the above ' 'requirements:\n' ) for req in [req for req in self.requirements.values() if req.comes_from]: parts.append('%s==%s\n' % (req.name, req.installed_version)) # FIXME: should we do something with self.unnamed_requirements? return ''.join(parts) def _clean_zip_name(self, name, prefix): assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix) ) name = name[len(prefix) + 1:] name = name.replace(os.path.sep, '/') return name