-
-
Save sl5net/686de9621f85d8a652d66858c1b9dfb1 to your computer and use it in GitHub Desktop.
FLASK_APP=msFlaskApplication.py # FLASK_APP=app:create_app() # or app.py | |
FLASK_ENV=development | |
FLASK_DEBUG=0 | |
# FLASK_RUN_EXTRA_FILES= | |
# FLASK_RUN_HOST= | |
# FLASK_RUN_PORT=8080 | |
# FLASK_RUN_CERT= | |
# FLASK_RUN_KEY= |
# Default ignored files | |
/shelf/ | |
/workspace.xml |
<?xml version="1.0" encoding="UTF-8"?> | |
<module type="PYTHON_MODULE" version="4"> | |
<component name="NewModuleRootManager"> | |
<content url="file://$MODULE_DIR$" /> | |
<orderEntry type="inheritedJdk" /> | |
<orderEntry type="sourceFolder" forTests="false" /> | |
</component> | |
</module> |
<component name="InspectionProjectProfileManager"> | |
<settings> | |
<option name="USE_PROJECT_PROFILE" value="false" /> | |
<version value="1.0" /> | |
</settings> | |
</component> |
<?xml version="1.0" encoding="UTF-8"?> | |
<project version="4"> | |
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8" project-jdk-type="Python SDK" /> | |
<component name="PyCharmProfessionalAdvertiser"> | |
<option name="shown" value="true" /> | |
</component> | |
</project> |
<?xml version="1.0" encoding="UTF-8"?> | |
<project version="4"> | |
<component name="ProjectModuleManager"> | |
<modules> | |
<module fileurl="file://$PROJECT_DIR$/.idea/flask-hello-worl.iml" filepath="$PROJECT_DIR$/.idea/flask-hello-worl.iml" /> | |
</modules> | |
</component> | |
</project> |
<?xml version="1.0" encoding="UTF-8"?> | |
<project version="4"> | |
<component name="ChangeListManager"> | |
<list default="true" id="475bf091-af62-40aa-89fb-220ad4d5cc16" name="Default Changelist" comment="" /> | |
<option name="SHOW_DIALOG" value="false" /> | |
<option name="HIGHLIGHT_CONFLICTS" value="true" /> | |
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> | |
<option name="LAST_RESOLUTION" value="IGNORE" /> | |
</component> | |
<component name="FileTemplateManagerImpl"> | |
<option name="RECENT_TEMPLATES"> | |
<list> | |
<option value="Python Script" /> | |
</list> | |
</option> | |
</component> | |
<component name="ProjectId" id="1mc1T5SqaT65vWT7bEJe9FYKIdl" /> | |
<component name="ProjectViewState"> | |
<option name="autoscrollFromSource" value="true" /> | |
<option name="hideEmptyMiddlePackages" value="true" /> | |
<option name="showLibraryContents" value="true" /> | |
</component> | |
<component name="PropertiesComponent"> | |
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" /> | |
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" /> | |
<property name="WebServerToolWindowFactoryState" value="false" /> | |
<property name="last_opened_file_path" value="$PROJECT_DIR$" /> | |
</component> | |
<component name="RunManager"> | |
<configuration name="hello" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true"> | |
<module name="flask-hello-worl" /> | |
<option name="INTERPRETER_OPTIONS" value="" /> | |
<option name="PARENT_ENVS" value="true" /> | |
<envs> | |
<env name="PYTHONUNBUFFERED" value="1" /> | |
</envs> | |
<option name="SDK_HOME" value="" /> | |
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" /> | |
<option name="IS_MODULE_SDK" value="true" /> | |
<option name="ADD_CONTENT_ROOTS" value="true" /> | |
<option name="ADD_SOURCE_ROOTS" value="true" /> | |
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" /> | |
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/hello.py" /> | |
<option name="PARAMETERS" value="" /> | |
<option name="SHOW_COMMAND_LINE" value="false" /> | |
<option name="EMULATE_TERMINAL" value="false" /> | |
<option name="MODULE_MODE" value="false" /> | |
<option name="REDIRECT_INPUT" value="false" /> | |
<option name="INPUT_FILE" value="" /> | |
<method v="2" /> | |
</configuration> | |
</component> | |
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" /> | |
<component name="TaskManager"> | |
<task active="true" id="Default" summary="Default task"> | |
<changelist id="475bf091-af62-40aa-89fb-220ad4d5cc16" name="Default Changelist" comment="" /> | |
<created>1609778069486</created> | |
<option name="number" value="Default" /> | |
<option name="presentableId" value="Default" /> | |
<updated>1609778069486</updated> | |
<workItem from="1609778369019" duration="4934000" /> | |
</task> | |
<servers /> | |
</component> | |
<component name="TypeScriptGeneratedFilesManager"> | |
<option name="version" value="3" /> | |
</component> | |
</project> |
cd ~/PycharmProjects/flask-hello-worl ; . venv/bin/activate | |
(venv) administrator@kubuntu-MS-7C37:~/PycharmProjects/flask-hello-worl$ flask shell * Tip: There are .env or .flaskenv files present. Do "pip install python-dotenv" to use them. | |
Usage: flask shell [OPTIONS] | |
Error: Could not locate a Flask application. You did not provide the "FLASK_APP" environment variable, and a "wsgi.py" or "app.py" module was not found in the current directory. | |
(venv) administrator@kubuntu-MS-7C37:~/PycharmProjects/flask-hello-worl$ |
from flask import Flask | |
app = Flask(__name__) | |
@app.route('/') | |
def hello(): | |
return 'Hello, World!' | |
from flask import Flask | |
from flask_sqlalchemy import SQLAlchemy | |
app = Flask(__name__) | |
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/test.db' | |
db = SQLAlchemy(app) | |
class User(db.Model): | |
id = db.Column(db.Integer, primary_key=True) | |
username = db.Column(db.String(80), unique=True, nullable=False) | |
email = db.Column(db.String(120), unique=True, nullable=False) | |
def __repr__(self): | |
return '<User %r>' % self.username | |
# This file must be used with "source bin/activate" *from bash* | |
# you cannot run it directly | |
deactivate () { | |
# reset old environment variables | |
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then | |
PATH="${_OLD_VIRTUAL_PATH:-}" | |
export PATH | |
unset _OLD_VIRTUAL_PATH | |
fi | |
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then | |
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" | |
export PYTHONHOME | |
unset _OLD_VIRTUAL_PYTHONHOME | |
fi | |
# This should detect bash and zsh, which have a hash command that must | |
# be called to get it to forget past commands. Without forgetting | |
# past commands the $PATH changes we made may not be respected | |
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then | |
hash -r | |
fi | |
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then | |
PS1="${_OLD_VIRTUAL_PS1:-}" | |
export PS1 | |
unset _OLD_VIRTUAL_PS1 | |
fi | |
unset VIRTUAL_ENV | |
if [ ! "${1:-}" = "nondestructive" ] ; then | |
# Self destruct! | |
unset -f deactivate | |
fi | |
} | |
# unset irrelevant variables | |
deactivate nondestructive | |
VIRTUAL_ENV="/home/administrator/PycharmProjects/flask-hello-worl/venv" | |
export VIRTUAL_ENV | |
_OLD_VIRTUAL_PATH="$PATH" | |
PATH="$VIRTUAL_ENV/bin:$PATH" | |
export PATH | |
# unset PYTHONHOME if set | |
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) | |
# could use `if (set -u; : $PYTHONHOME) ;` in bash | |
if [ -n "${PYTHONHOME:-}" ] ; then | |
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" | |
unset PYTHONHOME | |
fi | |
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then | |
_OLD_VIRTUAL_PS1="${PS1:-}" | |
if [ "x(venv) " != x ] ; then | |
PS1="(venv) ${PS1:-}" | |
else | |
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then | |
# special case for Aspen magic directories | |
# see http://www.zetadev.com/software/aspen/ | |
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" | |
else | |
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" | |
fi | |
fi | |
export PS1 | |
fi | |
# This should detect bash and zsh, which have a hash command that must | |
# be called to get it to forget past commands. Without forgetting | |
# past commands the $PATH changes we made may not be respected | |
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then | |
hash -r | |
fi |
# This file must be used with "source bin/activate.csh" *from csh*. | |
# You cannot run it directly. | |
# Created by Davide Di Blasi <[email protected]>. | |
# Ported to Python 3.3 venv by Andrew Svetlov <[email protected]> | |
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' | |
# Unset irrelevant variables. | |
deactivate nondestructive | |
setenv VIRTUAL_ENV "/home/administrator/PycharmProjects/flask-hello-worl/venv" | |
set _OLD_VIRTUAL_PATH="$PATH" | |
setenv PATH "$VIRTUAL_ENV/bin:$PATH" | |
set _OLD_VIRTUAL_PROMPT="$prompt" | |
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then | |
if ("venv" != "") then | |
set env_name = "venv" | |
else | |
if (`basename "VIRTUAL_ENV"` == "__") then | |
# special case for Aspen magic directories | |
# see http://www.zetadev.com/software/aspen/ | |
set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` | |
else | |
set env_name = `basename "$VIRTUAL_ENV"` | |
endif | |
endif | |
set prompt = "[$env_name] $prompt" | |
unset env_name | |
endif | |
alias pydoc python -m pydoc | |
rehash |
# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) | |
# you cannot run it directly | |
function deactivate -d "Exit virtualenv and return to normal shell environment" | |
# reset old environment variables | |
if test -n "$_OLD_VIRTUAL_PATH" | |
set -gx PATH $_OLD_VIRTUAL_PATH | |
set -e _OLD_VIRTUAL_PATH | |
end | |
if test -n "$_OLD_VIRTUAL_PYTHONHOME" | |
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME | |
set -e _OLD_VIRTUAL_PYTHONHOME | |
end | |
if test -n "$_OLD_FISH_PROMPT_OVERRIDE" | |
functions -e fish_prompt | |
set -e _OLD_FISH_PROMPT_OVERRIDE | |
functions -c _old_fish_prompt fish_prompt | |
functions -e _old_fish_prompt | |
end | |
set -e VIRTUAL_ENV | |
if test "$argv[1]" != "nondestructive" | |
# Self destruct! | |
functions -e deactivate | |
end | |
end | |
# unset irrelevant variables | |
deactivate nondestructive | |
set -gx VIRTUAL_ENV "/home/administrator/PycharmProjects/flask-hello-worl/venv" | |
set -gx _OLD_VIRTUAL_PATH $PATH | |
set -gx PATH "$VIRTUAL_ENV/bin" $PATH | |
# unset PYTHONHOME if set | |
if set -q PYTHONHOME | |
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME | |
set -e PYTHONHOME | |
end | |
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" | |
# fish uses a function instead of an env var to generate the prompt. | |
# save the current fish_prompt function as the function _old_fish_prompt | |
functions -c fish_prompt _old_fish_prompt | |
# with the original prompt function renamed, we can override with our own. | |
function fish_prompt | |
# Save the return status of the last command | |
set -l old_status $status | |
# Prompt override? | |
if test -n "(venv) " | |
printf "%s%s" "(venv) " (set_color normal) | |
else | |
# ...Otherwise, prepend env | |
set -l _checkbase (basename "$VIRTUAL_ENV") | |
if test $_checkbase = "__" | |
# special case for Aspen magic directories | |
# see http://www.zetadev.com/software/aspen/ | |
printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) | |
else | |
printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) | |
end | |
end | |
# Restore the return status of the previous command. | |
echo "exit $old_status" | . | |
_old_fish_prompt | |
end | |
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" | |
end |
<# | |
.Synopsis | |
Activate a Python virtual environment for the current PowerShell session. | |
.Description | |
Pushes the python executable for a virtual environment to the front of the | |
$Env:PATH environment variable and sets the prompt to signify that you are | |
in a Python virtual environment. Makes use of the command line switches as | |
well as the `pyvenv.cfg` file values present in the virtual environment. | |
.Parameter VenvDir | |
Path to the directory that contains the virtual environment to activate. The | |
default value for this is the parent of the directory that the Activate.ps1 | |
script is located within. | |
.Parameter Prompt | |
The prompt prefix to display when this virtual environment is activated. By | |
default, this prompt is the name of the virtual environment folder (VenvDir) | |
surrounded by parentheses and followed by a single space (ie. '(.venv) '). | |
.Example | |
Activate.ps1 | |
Activates the Python virtual environment that contains the Activate.ps1 script. | |
.Example | |
Activate.ps1 -Verbose | |
Activates the Python virtual environment that contains the Activate.ps1 script, | |
and shows extra information about the activation as it executes. | |
.Example | |
Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv | |
Activates the Python virtual environment located in the specified location. | |
.Example | |
Activate.ps1 -Prompt "MyPython" | |
Activates the Python virtual environment that contains the Activate.ps1 script, | |
and prefixes the current prompt with the specified string (surrounded in | |
parentheses) while the virtual environment is active. | |
.Notes | |
On Windows, it may be required to enable this Activate.ps1 script by setting the | |
execution policy for the user. You can do this by issuing the following PowerShell | |
command: | |
PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser | |
For more information on Execution Policies: | |
https://go.microsoft.com/fwlink/?LinkID=135170 | |
#> | |
Param( | |
[Parameter(Mandatory = $false)] | |
[String] | |
$VenvDir, | |
[Parameter(Mandatory = $false)] | |
[String] | |
$Prompt | |
) | |
<# Function declarations --------------------------------------------------- #> | |
<# | |
.Synopsis | |
Remove all shell session elements added by the Activate script, including the | |
addition of the virtual environment's Python executable from the beginning of | |
the PATH variable. | |
.Parameter NonDestructive | |
If present, do not remove this function from the global namespace for the | |
session. | |
#> | |
function global:deactivate ([switch]$NonDestructive) { | |
# Revert to original values | |
# The prior prompt: | |
if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { | |
Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt | |
Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT | |
} | |
# The prior PYTHONHOME: | |
if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { | |
Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME | |
Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME | |
} | |
# The prior PATH: | |
if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { | |
Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH | |
Remove-Item -Path Env:_OLD_VIRTUAL_PATH | |
} | |
# Just remove the VIRTUAL_ENV altogether: | |
if (Test-Path -Path Env:VIRTUAL_ENV) { | |
Remove-Item -Path env:VIRTUAL_ENV | |
} | |
# Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: | |
if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { | |
Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force | |
} | |
# Leave deactivate function in the global namespace if requested: | |
if (-not $NonDestructive) { | |
Remove-Item -Path function:deactivate | |
} | |
} | |
<# | |
.Description | |
Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the | |
given folder, and returns them in a map. | |
For each line in the pyvenv.cfg file, if that line can be parsed into exactly | |
two strings separated by `=` (with any amount of whitespace surrounding the =) | |
then it is considered a `key = value` line. The left hand string is the key, | |
the right hand is the value. | |
If the value starts with a `'` or a `"` then the first and last character is | |
stripped from the value before being captured. | |
.Parameter ConfigDir | |
Path to the directory that contains the `pyvenv.cfg` file. | |
#> | |
function Get-PyVenvConfig( | |
[String] | |
$ConfigDir | |
) { | |
Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" | |
# Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). | |
$pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue | |
# An empty map will be returned if no config file is found. | |
$pyvenvConfig = @{ } | |
if ($pyvenvConfigPath) { | |
Write-Verbose "File exists, parse `key = value` lines" | |
$pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath | |
$pyvenvConfigContent | ForEach-Object { | |
$keyval = $PSItem -split "\s*=\s*", 2 | |
if ($keyval[0] -and $keyval[1]) { | |
$val = $keyval[1] | |
# Remove extraneous quotations around a string value. | |
if ("'""".Contains($val.Substring(0, 1))) { | |
$val = $val.Substring(1, $val.Length - 2) | |
} | |
$pyvenvConfig[$keyval[0]] = $val | |
Write-Verbose "Adding Key: '$($keyval[0])'='$val'" | |
} | |
} | |
} | |
return $pyvenvConfig | |
} | |
<# Begin Activate script --------------------------------------------------- #> | |
# Determine the containing directory of this script | |
$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition | |
$VenvExecDir = Get-Item -Path $VenvExecPath | |
Write-Verbose "Activation script is located in path: '$VenvExecPath'" | |
Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" | |
Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" | |
# Set values required in priority: CmdLine, ConfigFile, Default | |
# First, get the location of the virtual environment, it might not be | |
# VenvExecDir if specified on the command line. | |
if ($VenvDir) { | |
Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" | |
} | |
else { | |
Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." | |
$VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") | |
Write-Verbose "VenvDir=$VenvDir" | |
} | |
# Next, read the `pyvenv.cfg` file to determine any required value such | |
# as `prompt`. | |
$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir | |
# Next, set the prompt from the command line, or the config file, or | |
# just use the name of the virtual environment folder. | |
if ($Prompt) { | |
Write-Verbose "Prompt specified as argument, using '$Prompt'" | |
} | |
else { | |
Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" | |
if ($pyvenvCfg -and $pyvenvCfg['prompt']) { | |
Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" | |
$Prompt = $pyvenvCfg['prompt']; | |
} | |
else { | |
Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virutal environment)" | |
Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" | |
$Prompt = Split-Path -Path $venvDir -Leaf | |
} | |
} | |
Write-Verbose "Prompt = '$Prompt'" | |
Write-Verbose "VenvDir='$VenvDir'" | |
# Deactivate any currently active virtual environment, but leave the | |
# deactivate function in place. | |
deactivate -nondestructive | |
# Now set the environment variable VIRTUAL_ENV, used by many tools to determine | |
# that there is an activated venv. | |
$env:VIRTUAL_ENV = $VenvDir | |
if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { | |
Write-Verbose "Setting prompt to '$Prompt'" | |
# Set the prompt to include the env name | |
# Make sure _OLD_VIRTUAL_PROMPT is global | |
function global:_OLD_VIRTUAL_PROMPT { "" } | |
Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT | |
New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt | |
function global:prompt { | |
Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " | |
_OLD_VIRTUAL_PROMPT | |
} | |
} | |
# Clear PYTHONHOME | |
if (Test-Path -Path Env:PYTHONHOME) { | |
Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME | |
Remove-Item -Path Env:PYTHONHOME | |
} | |
# Add the venv to the PATH | |
Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH | |
$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from dotenv.cli import cli | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(cli()) |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from setuptools.command.easy_install import main | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(main()) |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from setuptools.command.easy_install import main | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(main()) |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from pip._internal.cli.main import main | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(main()) |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from pip._internal.cli.main import main | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(main()) |
#!/home/administrator/PycharmProjects/flask-hello-worl/venv/bin/python3 | |
# -*- coding: utf-8 -*- | |
import re | |
import sys | |
from pip._internal.cli.main import main | |
if __name__ == '__main__': | |
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) | |
sys.exit(main()) |
from .compat import IS_TYPE_CHECKING | |
from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values | |
if IS_TYPE_CHECKING: | |
from typing import Any, Optional | |
def load_ipython_extension(ipython): | |
# type: (Any) -> None | |
from .ipython import load_ipython_extension | |
load_ipython_extension(ipython) | |
def get_cli_string(path=None, action=None, key=None, value=None, quote=None): | |
# type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> str | |
"""Returns a string suitable for running as a shell script. | |
Useful for converting a arguments passed to a fabric task | |
to be passed to a `local` or `run` command. | |
""" | |
command = ['dotenv'] | |
if quote: | |
command.append('-q %s' % quote) | |
if path: | |
command.append('-f %s' % path) | |
if action: | |
command.append(action) | |
if key: | |
command.append(key) | |
if value: | |
if ' ' in value: | |
command.append('"%s"' % value) | |
else: | |
command.append(value) | |
return ' '.join(command).strip() | |
__all__ = ['get_cli_string', | |
'load_dotenv', | |
'dotenv_values', | |
'get_key', | |
'set_key', | |
'unset_key', | |
'find_dotenv', | |
'load_ipython_extension'] |
import os | |
import sys | |
from subprocess import Popen | |
try: | |
import click | |
except ImportError: | |
sys.stderr.write('It seems python-dotenv is not installed with cli option. \n' | |
'Run pip install "python-dotenv[cli]" to fix this.') | |
sys.exit(1) | |
from .compat import IS_TYPE_CHECKING, to_env | |
from .main import dotenv_values, get_key, set_key, unset_key | |
from .version import __version__ | |
if IS_TYPE_CHECKING: | |
from typing import Any, List, Dict | |
@click.group() | |
@click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), | |
type=click.Path(file_okay=True), | |
help="Location of the .env file, defaults to .env file in current working directory.") | |
@click.option('-q', '--quote', default='always', | |
type=click.Choice(['always', 'never', 'auto']), | |
help="Whether to quote or not the variable values. Default mode is always. This does not affect parsing.") | |
@click.option('-e', '--export', default=False, | |
type=click.BOOL, | |
help="Whether to write the dot file as an executable bash script.") | |
@click.version_option(version=__version__) | |
@click.pass_context | |
def cli(ctx, file, quote, export): | |
# type: (click.Context, Any, Any, Any) -> None | |
'''This script is used to set, get or unset values from a .env file.''' | |
ctx.obj = {} | |
ctx.obj['QUOTE'] = quote | |
ctx.obj['EXPORT'] = export | |
ctx.obj['FILE'] = file | |
@cli.command() | |
@click.pass_context | |
def list(ctx): | |
# type: (click.Context) -> None | |
'''Display all the stored key/value.''' | |
file = ctx.obj['FILE'] | |
if not os.path.isfile(file): | |
raise click.BadParameter( | |
'Path "%s" does not exist.' % (file), | |
ctx=ctx | |
) | |
dotenv_as_dict = dotenv_values(file) | |
for k, v in dotenv_as_dict.items(): | |
click.echo('%s=%s' % (k, v)) | |
@cli.command() | |
@click.pass_context | |
@click.argument('key', required=True) | |
@click.argument('value', required=True) | |
def set(ctx, key, value): | |
# type: (click.Context, Any, Any) -> None | |
'''Store the given key/value.''' | |
file = ctx.obj['FILE'] | |
quote = ctx.obj['QUOTE'] | |
export = ctx.obj['EXPORT'] | |
success, key, value = set_key(file, key, value, quote, export) | |
if success: | |
click.echo('%s=%s' % (key, value)) | |
else: | |
exit(1) | |
@cli.command() | |
@click.pass_context | |
@click.argument('key', required=True) | |
def get(ctx, key): | |
# type: (click.Context, Any) -> None | |
'''Retrieve the value for the given key.''' | |
file = ctx.obj['FILE'] | |
if not os.path.isfile(file): | |
raise click.BadParameter( | |
'Path "%s" does not exist.' % (file), | |
ctx=ctx | |
) | |
stored_value = get_key(file, key) | |
if stored_value: | |
click.echo('%s=%s' % (key, stored_value)) | |
else: | |
exit(1) | |
@cli.command() | |
@click.pass_context | |
@click.argument('key', required=True) | |
def unset(ctx, key): | |
# type: (click.Context, Any) -> None | |
'''Removes the given key.''' | |
file = ctx.obj['FILE'] | |
quote = ctx.obj['QUOTE'] | |
success, key = unset_key(file, key, quote) | |
if success: | |
click.echo("Successfully removed %s" % key) | |
else: | |
exit(1) | |
@cli.command(context_settings={'ignore_unknown_options': True}) | |
@click.pass_context | |
@click.argument('commandline', nargs=-1, type=click.UNPROCESSED) | |
def run(ctx, commandline): | |
# type: (click.Context, List[str]) -> None | |
"""Run command with environment variables present.""" | |
file = ctx.obj['FILE'] | |
if not os.path.isfile(file): | |
raise click.BadParameter( | |
'Invalid value for \'-f\' "%s" does not exist.' % (file), | |
ctx=ctx | |
) | |
dotenv_as_dict = {to_env(k): to_env(v) for (k, v) in dotenv_values(file).items() if v is not None} | |
if not commandline: | |
click.echo('No command given.') | |
exit(1) | |
ret = run_command(commandline, dotenv_as_dict) | |
exit(ret) | |
def run_command(command, env): | |
# type: (List[str], Dict[str, str]) -> int | |
"""Run command in sub process. | |
Runs the command in a sub process with the variables from `env` | |
added in the current environment variables. | |
Parameters | |
---------- | |
command: List[str] | |
The command and it's parameters | |
env: Dict | |
The additional environment variables | |
Returns | |
------- | |
int | |
The return code of the command | |
""" | |
# copy the current environment variables and add the vales from | |
# `env` | |
cmd_env = os.environ.copy() | |
cmd_env.update(env) | |
p = Popen(command, | |
universal_newlines=True, | |
bufsize=0, | |
shell=False, | |
env=cmd_env) | |
_, _ = p.communicate() | |
return p.returncode | |
if __name__ == "__main__": | |
cli() |
import sys | |
PY2 = sys.version_info[0] == 2 # type: bool | |
if PY2: | |
from StringIO import StringIO # noqa | |
else: | |
from io import StringIO # noqa | |
def is_type_checking(): | |
# type: () -> bool | |
try: | |
from typing import TYPE_CHECKING | |
except ImportError: | |
return False | |
return TYPE_CHECKING | |
IS_TYPE_CHECKING = is_type_checking() | |
if IS_TYPE_CHECKING: | |
from typing import Text | |
def to_env(text): | |
# type: (Text) -> str | |
""" | |
Encode a string the same way whether it comes from the environment or a `.env` file. | |
""" | |
if PY2: | |
return text.encode(sys.getfilesystemencoding() or "utf-8") | |
else: | |
return text | |
def to_text(string): | |
# type: (str) -> Text | |
""" | |
Make a string Unicode if it isn't already. | |
This is useful for defining raw unicode strings because `ur"foo"` isn't valid in | |
Python 3. | |
""" | |
if PY2: | |
return string.decode("utf-8") | |
else: | |
return string |
from __future__ import print_function | |
from IPython.core.magic import Magics, line_magic, magics_class # type: ignore | |
from IPython.core.magic_arguments import (argument, magic_arguments, # type: ignore | |
parse_argstring) # type: ignore | |
from .main import find_dotenv, load_dotenv | |
@magics_class | |
class IPythonDotEnv(Magics): | |
@magic_arguments() | |
@argument( | |
'-o', '--override', action='store_true', | |
help="Indicate to override existing variables" | |
) | |
@argument( | |
'-v', '--verbose', action='store_true', | |
help="Indicate function calls to be verbose" | |
) | |
@argument('dotenv_path', nargs='?', type=str, default='.env', | |
help='Search in increasingly higher folders for the `dotenv_path`') | |
@line_magic | |
def dotenv(self, line): | |
args = parse_argstring(self.dotenv, line) | |
# Locate the .env file | |
dotenv_path = args.dotenv_path | |
try: | |
dotenv_path = find_dotenv(dotenv_path, True, True) | |
except IOError: | |
print("cannot find .env file") | |
return | |
# Load the .env file | |
load_dotenv(dotenv_path, verbose=args.verbose, override=args.override) | |
def load_ipython_extension(ipython): | |
"""Register the %dotenv magic.""" | |
ipython.register_magics(IPythonDotEnv) |
# -*- coding: utf-8 -*- | |
from __future__ import absolute_import, print_function, unicode_literals | |
import io | |
import logging | |
import os | |
import re | |
import shutil | |
import sys | |
import tempfile | |
from collections import OrderedDict | |
from contextlib import contextmanager | |
from .compat import IS_TYPE_CHECKING, PY2, StringIO, to_env | |
from .parser import Binding, parse_stream | |
logger = logging.getLogger(__name__) | |
if IS_TYPE_CHECKING: | |
from typing import ( | |
Dict, Iterable, Iterator, Match, Optional, Pattern, Union, Text, IO, Tuple | |
) | |
if sys.version_info >= (3, 6): | |
_PathLike = os.PathLike | |
else: | |
_PathLike = Text | |
if sys.version_info >= (3, 0): | |
_StringIO = StringIO | |
else: | |
_StringIO = StringIO[Text] | |
__posix_variable = re.compile( | |
r""" | |
\$\{ | |
(?P<name>[^\}:]*) | |
(?::- | |
(?P<default>[^\}]*) | |
)? | |
\} | |
""", | |
re.VERBOSE, | |
) # type: Pattern[Text] | |
def with_warn_for_invalid_lines(mappings): | |
# type: (Iterator[Binding]) -> Iterator[Binding] | |
for mapping in mappings: | |
if mapping.error: | |
logger.warning( | |
"Python-dotenv could not parse statement starting at line %s", | |
mapping.original.line, | |
) | |
yield mapping | |
class DotEnv(): | |
def __init__(self, dotenv_path, verbose=False, encoding=None, interpolate=True): | |
# type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text], bool) -> None | |
self.dotenv_path = dotenv_path # type: Union[Text,_PathLike, _StringIO] | |
self._dict = None # type: Optional[Dict[Text, Optional[Text]]] | |
self.verbose = verbose # type: bool | |
self.encoding = encoding # type: Union[None, Text] | |
self.interpolate = interpolate # type: bool | |
@contextmanager | |
def _get_stream(self): | |
# type: () -> Iterator[IO[Text]] | |
if isinstance(self.dotenv_path, StringIO): | |
yield self.dotenv_path | |
elif os.path.isfile(self.dotenv_path): | |
with io.open(self.dotenv_path, encoding=self.encoding) as stream: | |
yield stream | |
else: | |
if self.verbose: | |
logger.info("Python-dotenv could not find configuration file %s.", self.dotenv_path or '.env') | |
yield StringIO('') | |
def dict(self): | |
# type: () -> Dict[Text, Optional[Text]] | |
"""Return dotenv as dict""" | |
if self._dict: | |
return self._dict | |
if self.interpolate: | |
values = resolve_nested_variables(self.parse()) | |
else: | |
values = OrderedDict(self.parse()) | |
self._dict = values | |
return values | |
def parse(self): | |
# type: () -> Iterator[Tuple[Text, Optional[Text]]] | |
with self._get_stream() as stream: | |
for mapping in with_warn_for_invalid_lines(parse_stream(stream)): | |
if mapping.key is not None: | |
yield mapping.key, mapping.value | |
def set_as_environment_variables(self, override=False): | |
# type: (bool) -> bool | |
""" | |
Load the current dotenv as system environemt variable. | |
""" | |
for k, v in self.dict().items(): | |
if k in os.environ and not override: | |
continue | |
if v is not None: | |
os.environ[to_env(k)] = to_env(v) | |
return True | |
def get(self, key): | |
# type: (Text) -> Optional[Text] | |
""" | |
""" | |
data = self.dict() | |
if key in data: | |
return data[key] | |
if self.verbose: | |
logger.warning("Key %s not found in %s.", key, self.dotenv_path) | |
return None | |
def get_key(dotenv_path, key_to_get): | |
# type: (Union[Text, _PathLike], Text) -> Optional[Text] | |
""" | |
Gets the value of a given key from the given .env | |
If the .env path given doesn't exist, fails | |
""" | |
return DotEnv(dotenv_path, verbose=True).get(key_to_get) | |
@contextmanager | |
def rewrite(path): | |
# type: (_PathLike) -> Iterator[Tuple[IO[Text], IO[Text]]] | |
try: | |
if not os.path.isfile(path): | |
with io.open(path, "w+") as source: | |
source.write("") | |
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest: | |
with io.open(path) as source: | |
yield (source, dest) # type: ignore | |
except BaseException: | |
if os.path.isfile(dest.name): | |
os.unlink(dest.name) | |
raise | |
else: | |
shutil.move(dest.name, path) | |
def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always", export=False): | |
# type: (_PathLike, Text, Text, Text, bool) -> Tuple[Optional[bool], Text, Text] | |
""" | |
Adds or Updates a key/value to the given .env | |
If the .env path given doesn't exist, fails instead of risking creating | |
an orphan .env somewhere in the filesystem | |
""" | |
value_to_set = value_to_set.strip("'").strip('"') | |
if " " in value_to_set: | |
quote_mode = "always" | |
if quote_mode == "always": | |
value_out = '"{}"'.format(value_to_set.replace('"', '\\"')) | |
else: | |
value_out = value_to_set | |
if export: | |
line_out = 'export {}={}\n'.format(key_to_set, value_out) | |
else: | |
line_out = "{}={}\n".format(key_to_set, value_out) | |
with rewrite(dotenv_path) as (source, dest): | |
replaced = False | |
for mapping in with_warn_for_invalid_lines(parse_stream(source)): | |
if mapping.key == key_to_set: | |
dest.write(line_out) | |
replaced = True | |
else: | |
dest.write(mapping.original.string) | |
if not replaced: | |
dest.write(line_out) | |
return True, key_to_set, value_to_set | |
def unset_key(dotenv_path, key_to_unset, quote_mode="always"): | |
# type: (_PathLike, Text, Text) -> Tuple[Optional[bool], Text] | |
""" | |
Removes a given key from the given .env | |
If the .env path given doesn't exist, fails | |
If the given key doesn't exist in the .env, fails | |
""" | |
if not os.path.exists(dotenv_path): | |
logger.warning("Can't delete from %s - it doesn't exist.", dotenv_path) | |
return None, key_to_unset | |
removed = False | |
with rewrite(dotenv_path) as (source, dest): | |
for mapping in with_warn_for_invalid_lines(parse_stream(source)): | |
if mapping.key == key_to_unset: | |
removed = True | |
else: | |
dest.write(mapping.original.string) | |
if not removed: | |
logger.warning("Key %s not removed from %s - key doesn't exist.", key_to_unset, dotenv_path) | |
return None, key_to_unset | |
return removed, key_to_unset | |
def resolve_nested_variables(values): | |
# type: (Iterable[Tuple[Text, Optional[Text]]]) -> Dict[Text, Optional[Text]] | |
def _replacement(name, default): | |
# type: (Text, Optional[Text]) -> Text | |
default = default if default is not None else "" | |
ret = new_values.get(name, os.getenv(name, default)) | |
return ret # type: ignore | |
def _re_sub_callback(match): | |
# type: (Match[Text]) -> Text | |
""" | |
From a match object gets the variable name and returns | |
the correct replacement | |
""" | |
matches = match.groupdict() | |
return _replacement(name=matches["name"], default=matches["default"]) # type: ignore | |
new_values = {} | |
for (k, v) in values: | |
new_values[k] = __posix_variable.sub(_re_sub_callback, v) if v is not None else None | |
return new_values | |
def _walk_to_root(path): | |
# type: (Text) -> Iterator[Text] | |
""" | |
Yield directories starting from the given directory up to the root | |
""" | |
if not os.path.exists(path): | |
raise IOError('Starting path not found') | |
if os.path.isfile(path): | |
path = os.path.dirname(path) | |
last_dir = None | |
current_dir = os.path.abspath(path) | |
while last_dir != current_dir: | |
yield current_dir | |
parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir)) | |
last_dir, current_dir = current_dir, parent_dir | |
def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): | |
# type: (Text, bool, bool) -> Text | |
""" | |
Search in increasingly higher folders for the given file | |
Returns path to the file if found, or an empty string otherwise | |
""" | |
def _is_interactive(): | |
""" Decide whether this is running in a REPL or IPython notebook """ | |
main = __import__('__main__', None, None, fromlist=['__file__']) | |
return not hasattr(main, '__file__') | |
if usecwd or _is_interactive() or getattr(sys, 'frozen', False): | |
# Should work without __file__, e.g. in REPL or IPython notebook. | |
path = os.getcwd() | |
else: | |
# will work for .py files | |
frame = sys._getframe() | |
# find first frame that is outside of this file | |
if PY2 and not __file__.endswith('.py'): | |
# in Python2 __file__ extension could be .pyc or .pyo (this doesn't account | |
# for edge case of Python compiled for non-standard extension) | |
current_file = __file__.rsplit('.', 1)[0] + '.py' | |
else: | |
current_file = __file__ | |
while frame.f_code.co_filename == current_file: | |
assert frame.f_back is not None | |
frame = frame.f_back | |
frame_filename = frame.f_code.co_filename | |
path = os.path.dirname(os.path.abspath(frame_filename)) | |
for dirname in _walk_to_root(path): | |
check_path = os.path.join(dirname, filename) | |
if os.path.isfile(check_path): | |
return check_path | |
if raise_error_if_not_found: | |
raise IOError('File not found') | |
return '' | |
def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, interpolate=True, **kwargs): | |
# type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, bool, Union[None, Text]) -> bool | |
"""Parse a .env file and then load all the variables found as environment variables. | |
- *dotenv_path*: absolute or relative path to .env file. | |
- *stream*: `StringIO` object with .env content. | |
- *verbose*: whether to output the warnings related to missing .env file etc. Defaults to `False`. | |
- *override*: where to override the system environment variables with the variables in `.env` file. | |
Defaults to `False`. | |
""" | |
f = dotenv_path or stream or find_dotenv() | |
return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).set_as_environment_variables(override=override) | |
def dotenv_values(dotenv_path=None, stream=None, verbose=False, interpolate=True, **kwargs): | |
# type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> Dict[Text, Optional[Text]] # noqa: E501 | |
f = dotenv_path or stream or find_dotenv() | |
return DotEnv(f, verbose=verbose, interpolate=interpolate, **kwargs).dict() |
import codecs | |
import re | |
from .compat import IS_TYPE_CHECKING, to_text | |
if IS_TYPE_CHECKING: | |
from typing import ( # noqa:F401 | |
IO, Iterator, Match, NamedTuple, Optional, Pattern, Sequence, Text, | |
Tuple | |
) | |
def make_regex(string, extra_flags=0): | |
# type: (str, int) -> Pattern[Text] | |
return re.compile(to_text(string), re.UNICODE | extra_flags) | |
_newline = make_regex(r"(\r\n|\n|\r)") | |
_multiline_whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE) | |
_whitespace = make_regex(r"[^\S\r\n]*") | |
_export = make_regex(r"(?:export[^\S\r\n]+)?") | |
_single_quoted_key = make_regex(r"'([^']+)'") | |
_unquoted_key = make_regex(r"([^=\#\s]+)") | |
_equal_sign = make_regex(r"(=[^\S\r\n]*)") | |
_single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'") | |
_double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"') | |
_unquoted_value = make_regex(r"([^\r\n]*)") | |
_comment = make_regex(r"(?:[^\S\r\n]*#[^\r\n]*)?") | |
_end_of_line = make_regex(r"[^\S\r\n]*(?:\r\n|\n|\r|$)") | |
_rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?") | |
_double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]") | |
_single_quote_escapes = make_regex(r"\\[\\']") | |
try: | |
# this is necessary because we only import these from typing | |
# when we are type checking, and the linter is upset if we | |
# re-import | |
import typing | |
Original = typing.NamedTuple( | |
"Original", | |
[ | |
("string", typing.Text), | |
("line", int), | |
], | |
) | |
Binding = typing.NamedTuple( | |
"Binding", | |
[ | |
("key", typing.Optional[typing.Text]), | |
("value", typing.Optional[typing.Text]), | |
("original", Original), | |
("error", bool), | |
], | |
) | |
except (ImportError, AttributeError): | |
from collections import namedtuple | |
Original = namedtuple( # type: ignore | |
"Original", | |
[ | |
"string", | |
"line", | |
], | |
) | |
Binding = namedtuple( # type: ignore | |
"Binding", | |
[ | |
"key", | |
"value", | |
"original", | |
"error", | |
], | |
) | |
class Position: | |
def __init__(self, chars, line): | |
# type: (int, int) -> None | |
self.chars = chars | |
self.line = line | |
@classmethod | |
def start(cls): | |
# type: () -> Position | |
return cls(chars=0, line=1) | |
def set(self, other): | |
# type: (Position) -> None | |
self.chars = other.chars | |
self.line = other.line | |
def advance(self, string): | |
# type: (Text) -> None | |
self.chars += len(string) | |
self.line += len(re.findall(_newline, string)) | |
class Error(Exception): | |
pass | |
class Reader: | |
def __init__(self, stream): | |
# type: (IO[Text]) -> None | |
self.string = stream.read() | |
self.position = Position.start() | |
self.mark = Position.start() | |
def has_next(self): | |
# type: () -> bool | |
return self.position.chars < len(self.string) | |
def set_mark(self): | |
# type: () -> None | |
self.mark.set(self.position) | |
def get_marked(self): | |
# type: () -> Original | |
return Original( | |
string=self.string[self.mark.chars:self.position.chars], | |
line=self.mark.line, | |
) | |
def peek(self, count): | |
# type: (int) -> Text | |
return self.string[self.position.chars:self.position.chars + count] | |
def read(self, count): | |
# type: (int) -> Text | |
result = self.string[self.position.chars:self.position.chars + count] | |
if len(result) < count: | |
raise Error("read: End of string") | |
self.position.advance(result) | |
return result | |
def read_regex(self, regex): | |
# type: (Pattern[Text]) -> Sequence[Text] | |
match = regex.match(self.string, self.position.chars) | |
if match is None: | |
raise Error("read_regex: Pattern not found") | |
self.position.advance(self.string[match.start():match.end()]) | |
return match.groups() | |
def decode_escapes(regex, string): | |
# type: (Pattern[Text], Text) -> Text | |
def decode_match(match): | |
# type: (Match[Text]) -> Text | |
return codecs.decode(match.group(0), 'unicode-escape') # type: ignore | |
return regex.sub(decode_match, string) | |
def parse_key(reader): | |
# type: (Reader) -> Optional[Text] | |
char = reader.peek(1) | |
if char == "#": | |
return None | |
elif char == "'": | |
(key,) = reader.read_regex(_single_quoted_key) | |
else: | |
(key,) = reader.read_regex(_unquoted_key) | |
return key | |
def parse_unquoted_value(reader): | |
# type: (Reader) -> Text | |
(part,) = reader.read_regex(_unquoted_value) | |
return re.sub(r"\s+#.*", "", part).rstrip() | |
def parse_value(reader): | |
# type: (Reader) -> Text | |
char = reader.peek(1) | |
if char == u"'": | |
(value,) = reader.read_regex(_single_quoted_value) | |
return decode_escapes(_single_quote_escapes, value) | |
elif char == u'"': | |
(value,) = reader.read_regex(_double_quoted_value) | |
return decode_escapes(_double_quote_escapes, value) | |
elif char in (u"", u"\n", u"\r"): | |
return u"" | |
else: | |
return parse_unquoted_value(reader) | |
def parse_binding(reader): | |
# type: (Reader) -> Binding | |
reader.set_mark() | |
try: | |
reader.read_regex(_multiline_whitespace) | |
if not reader.has_next(): | |
return Binding( | |
key=None, | |
value=None, | |
original=reader.get_marked(), | |
error=False, | |
) | |
reader.read_regex(_export) | |
key = parse_key(reader) | |
reader.read_regex(_whitespace) | |
if reader.peek(1) == "=": | |
reader.read_regex(_equal_sign) | |
value = parse_value(reader) # type: Optional[Text] | |
else: | |
value = None | |
reader.read_regex(_comment) | |
reader.read_regex(_end_of_line) | |
return Binding( | |
key=key, | |
value=value, | |
original=reader.get_marked(), | |
error=False, | |
) | |
except Error: | |
reader.read_regex(_rest_of_line) | |
return Binding( | |
key=None, | |
value=None, | |
original=reader.get_marked(), | |
error=True, | |
) | |
def parse_stream(stream): | |
# type: (IO[Text]) -> Iterator[Binding] | |
reader = Reader(stream) | |
while reader.has_next(): | |
yield parse_binding(reader) |
# Marker file for PEP 561 |
__version__ = "0.15.0" |
"""Run the EasyInstall command""" | |
if __name__ == '__main__': | |
from setuptools.command.easy_install import main | |
main() |
[console_scripts] | |
pip = pip._internal.cli.main:main | |
pip3 = pip._internal.cli.main:main | |
pip3.8 = pip._internal.cli.main:main | |
pip |
Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) | |
Permission is hereby granted, free of charge, to any person obtaining | |
a copy of this software and associated documentation files (the | |
"Software"), to deal in the Software without restriction, including | |
without limitation the rights to use, copy, modify, merge, publish, | |
distribute, sublicense, and/or sell copies of the Software, and to | |
permit persons to whom the Software is furnished to do so, subject to | |
the following conditions: | |
The above copyright notice and this permission notice shall be | |
included in all copies or substantial portions of the Software. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE | |
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION | |
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
Metadata-Version: 2.1 | |
Name: pip | |
Version: 20.0.2 | |
Summary: The PyPA recommended tool for installing Python packages. | |
Home-page: https://pip.pypa.io/ | |
Author: The pip developers | |
Author-email: [email protected] | |
License: MIT | |
Project-URL: Documentation, https://pip.pypa.io | |
Project-URL: Source, https://github.com/pypa/pip | |
Keywords: distutils easy_install egg setuptools wheel virtualenv | |
Platform: UNKNOWN | |
Classifier: Development Status :: 5 - Production/Stable | |
Classifier: Intended Audience :: Developers | |
Classifier: License :: OSI Approved :: MIT License | |
Classifier: Topic :: Software Development :: Build Tools | |
Classifier: Programming Language :: Python | |
Classifier: Programming Language :: Python :: 2 | |
Classifier: Programming Language :: Python :: 2.7 | |
Classifier: Programming Language :: Python :: 3 | |
Classifier: Programming Language :: Python :: 3.5 | |
Classifier: Programming Language :: Python :: 3.6 | |
Classifier: Programming Language :: Python :: 3.7 | |
Classifier: Programming Language :: Python :: 3.8 | |
Classifier: Programming Language :: Python :: Implementation :: CPython | |
Classifier: Programming Language :: Python :: Implementation :: PyPy | |
Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.* | |
pip - The Python Package Installer | |
================================== | |
.. image:: https://img.shields.io/pypi/v/pip.svg | |
:target: https://pypi.org/project/pip/ | |
.. image:: https://readthedocs.org/projects/pip/badge/?version=latest | |
:target: https://pip.pypa.io/en/latest | |
pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes. | |
Please take a look at our documentation for how to install and use pip: | |
* `Installation`_ | |
* `Usage`_ | |
Updates are released regularly, with a new version every 3 months. More details can be found in our documentation: | |
* `Release notes`_ | |
* `Release process`_ | |
If you find bugs, need help, or want to talk to the developers please use our mailing lists or chat rooms: | |
* `Issue tracking`_ | |
* `Discourse channel`_ | |
* `User IRC`_ | |
If you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms: | |
* `GitHub page`_ | |
* `Dev documentation`_ | |
* `Dev mailing list`_ | |
* `Dev IRC`_ | |
Code of Conduct | |
--------------- | |
Everyone interacting in the pip project's codebases, issue trackers, chat | |
rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. | |
.. _package installer: https://packaging.python.org/guides/tool-recommendations/ | |
.. _Python Package Index: https://pypi.org | |
.. _Installation: https://pip.pypa.io/en/stable/installing.html | |
.. _Usage: https://pip.pypa.io/en/stable/ | |
.. _Release notes: https://pip.pypa.io/en/stable/news.html | |
.. _Release process: https://pip.pypa.io/en/latest/development/release-process/ | |
.. _GitHub page: https://github.com/pypa/pip | |
.. _Dev documentation: https://pip.pypa.io/en/latest/development | |
.. _Issue tracking: https://github.com/pypa/pip/issues | |
.. _Discourse channel: https://discuss.python.org/c/packaging | |
.. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev | |
.. _User IRC: https://webchat.freenode.net/?channels=%23pypa | |
.. _Dev IRC: https://webchat.freenode.net/?channels=%23pypa-dev | |
.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ | |
../../../bin/pip,sha256=0XM5sbZSOiGLvafVKJVvxHvozgWis_Q-UMCtsqfFL_U,274 | |
../../../bin/pip3,sha256=0XM5sbZSOiGLvafVKJVvxHvozgWis_Q-UMCtsqfFL_U,274 | |
../../../bin/pip3.8,sha256=0XM5sbZSOiGLvafVKJVvxHvozgWis_Q-UMCtsqfFL_U,274 | |
pip-20.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 | |
pip-20.0.2.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090 | |
pip-20.0.2.dist-info/METADATA,sha256=MSgjT2JTt8usp4Hopp5AGEmc-7sKR2Jd7HTMJqCoRhw,3352 | |
pip-20.0.2.dist-info/RECORD,, | |
pip-20.0.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 | |
pip-20.0.2.dist-info/entry_points.txt,sha256=HtfDOwpUlr9s73jqLQ6wF9V0_0qvUXJwCBz7Vwx0Ue0,125 | |
pip-20.0.2.dist-info/top_level.txt,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 | |
pip/__init__.py,sha256=U1AM82iShMaw90K6Yq0Q2-AZ1EsOcqQLQRB-rxwFtII,455 | |
pip/__main__.py,sha256=NM95x7KuQr-lwPoTjAC0d_QzLJsJjpmAoxZg0mP8s98,632 | |
pip/__pycache__/__init__.cpython-38.pyc,, | |
pip/__pycache__/__main__.cpython-38.pyc,, | |
pip/_internal/__init__.py,sha256=j5fiII6yCeZjpW7_7wAVRMM4DwE-gyARGVU4yAADDeE,517 | |
pip/_internal/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/__pycache__/build_env.cpython-38.pyc,, | |
pip/_internal/__pycache__/cache.cpython-38.pyc,, | |
pip/_internal/__pycache__/configuration.cpython-38.pyc,, | |
pip/_internal/__pycache__/exceptions.cpython-38.pyc,, | |
pip/_internal/__pycache__/legacy_resolve.cpython-38.pyc,, | |
pip/_internal/__pycache__/locations.cpython-38.pyc,, | |
pip/_internal/__pycache__/main.cpython-38.pyc,, | |
pip/_internal/__pycache__/pep425tags.cpython-38.pyc,, | |
pip/_internal/__pycache__/pyproject.cpython-38.pyc,, | |
pip/_internal/__pycache__/self_outdated_check.cpython-38.pyc,, | |
pip/_internal/__pycache__/wheel_builder.cpython-38.pyc,, | |
pip/_internal/build_env.py,sha256=--aNgzIdYrCOclHMwoAdpclCpfdFE_jooRuCy5gczwg,7532 | |
pip/_internal/cache.py,sha256=16GrnDRLBQNlfKWIuIF6Sa-EFS78kez_w1WEjT3ykTI,11605 | |
pip/_internal/cli/__init__.py,sha256=FkHBgpxxb-_gd6r1FjnNhfMOzAUYyXoXKJ6abijfcFU,132 | |
pip/_internal/cli/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/autocompletion.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/base_command.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/cmdoptions.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/command_context.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/main.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/main_parser.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/parser.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/req_command.cpython-38.pyc,, | |
pip/_internal/cli/__pycache__/status_codes.cpython-38.pyc,, | |
pip/_internal/cli/autocompletion.py,sha256=ekGNtcDI0p7rFVc-7s4T9Tbss4Jgb7vsB649XJIblRg,6547 | |
pip/_internal/cli/base_command.py,sha256=v6yl5XNRqye8BT9ep8wvpMu6lylP_Hu6D95r_HqbpbQ,7948 | |
pip/_internal/cli/cmdoptions.py,sha256=f1TVHuu_fR3lLlMo6b367H_GsWFv26tLI9cAS-kZfE0,28114 | |
pip/_internal/cli/command_context.py,sha256=ygMVoTy2jpNilKT-6416gFSQpaBtrKRBbVbi2fy__EU,975 | |
pip/_internal/cli/main.py,sha256=8iq3bHe5lxJTB2EvKOqZ38NS0MmoS79_S1kgj4QuH8A,2610 | |
pip/_internal/cli/main_parser.py,sha256=W9OWeryh7ZkqELohaFh0Ko9sB98ZkSeDmnYbOZ1imBc,2819 | |
pip/_internal/cli/parser.py,sha256=O9djTuYQuSfObiY-NU6p4MJCfWsRUnDpE2YGA_fwols,9487 | |
pip/_internal/cli/req_command.py,sha256=pAUAglpTn0mUA6lRs7KN71yOm1KDabD0ySVTQTqWTSA,12463 | |
pip/_internal/cli/status_codes.py,sha256=F6uDG6Gj7RNKQJUDnd87QKqI16Us-t-B0wPF_4QMpWc,156 | |
pip/_internal/commands/__init__.py,sha256=uTSj58QlrSKeXqCUSdL-eAf_APzx5BHy1ABxb0j5ZNE,3714 | |
pip/_internal/commands/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/check.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/completion.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/configuration.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/debug.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/download.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/freeze.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/hash.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/help.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/install.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/list.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/search.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/show.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/uninstall.cpython-38.pyc,, | |
pip/_internal/commands/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/commands/check.py,sha256=mgLNYT3bd6Kmynwh4zzcBmVlFZ-urMo40jTgk6U405E,1505 | |
pip/_internal/commands/completion.py,sha256=UFQvq0Q4_B96z1bvnQyMOq82aPSu05RejbLmqeTZjC0,2975 | |
pip/_internal/commands/configuration.py,sha256=6riioZjMhsNSEct7dE-X8SobGodk3WERKJvuyjBje4Q,7226 | |
pip/_internal/commands/debug.py,sha256=a8llax2hRkxgK-tvwdJgaCaZCYPIx0fDvrlMDoYr8bQ,4209 | |
pip/_internal/commands/download.py,sha256=zX_0-IeFb4C8dxSmGHxk-6H5kehtyTSsdWpjNpAhSww,5007 | |
pip/_internal/commands/freeze.py,sha256=QS-4ib8jbKJ2wrDaDbTuyaB3Y_iJ5CQC2gAVHuAv9QU,3481 | |
pip/_internal/commands/hash.py,sha256=47teimfAPhpkaVbSDaafck51BT3XXYuL83lAqc5lOcE,1735 | |
pip/_internal/commands/help.py,sha256=Nhecq--ydFn80Gm1Zvbf9943EcRJfO0TnXUhsF0RO7s,1181 | |
pip/_internal/commands/install.py,sha256=T4P3J1rw7CQrZX4OUamtcoWMkTrJBfUe6gWpTfZW1bQ,27286 | |
pip/_internal/commands/list.py,sha256=2l0JiqHxjxDHNTCb2HZOjwwdo4duS1R0MsqZb6HSMKk,10660 | |
pip/_internal/commands/search.py,sha256=7Il8nKZ9mM7qF5jlnBoPvSIFY9f-0-5IbYoX3miTuZY,5148 | |
pip/_internal/commands/show.py,sha256=Vzsj2oX0JBl94MPyF3LV8YoMcigl8B2UsMM8zp0pH2s,6792 | |
pip/_internal/commands/uninstall.py,sha256=8mldFbrQecSoWDZRqxBgJkrlvx6Y9Iy7cs-2BIgtXt4,2983 | |
pip/_internal/commands/wheel.py,sha256=TMU5ZhjLo7BIZQApGPsYfoCsbGTnvP-N9jkgPJXhj1Y,7170 | |
pip/_internal/configuration.py,sha256=MgKrLFBJBkF3t2VJM4tvlnEspfSuS4scp_LhHWh53nY,14222 | |
pip/_internal/distributions/__init__.py,sha256=ECBUW5Gtu9TjJwyFLvim-i6kUMYVuikNh9I5asL6tbA,959 | |
pip/_internal/distributions/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/distributions/__pycache__/base.cpython-38.pyc,, | |
pip/_internal/distributions/__pycache__/installed.cpython-38.pyc,, | |
pip/_internal/distributions/__pycache__/sdist.cpython-38.pyc,, | |
pip/_internal/distributions/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/distributions/base.py,sha256=ruprpM_L2T2HNi3KLUHlbHimZ1sWVw-3Q0Lb8O7TDAI,1425 | |
pip/_internal/distributions/installed.py,sha256=YqlkBKr6TVP1MAYS6SG8ojud21wVOYLMZ8jMLJe9MSU,760 | |
pip/_internal/distributions/sdist.py,sha256=D4XTMlCwgPlK69l62GLYkNSVTVe99fR5iAcVt2EbGok,4086 | |
pip/_internal/distributions/wheel.py,sha256=95uD-TfaYoq3KiKBdzk9YMN4RRqJ28LNoSTS2K46gek,1294 | |
pip/_internal/exceptions.py,sha256=6YRuwXAK6F1iyUWKIkCIpWWN2khkAn1sZOgrFA9S8Ro,10247 | |
pip/_internal/index/__init__.py,sha256=vpt-JeTZefh8a-FC22ZeBSXFVbuBcXSGiILhQZJaNpQ,30 | |
pip/_internal/index/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/index/__pycache__/collector.cpython-38.pyc,, | |
pip/_internal/index/__pycache__/package_finder.cpython-38.pyc,, | |
pip/_internal/index/collector.py,sha256=YS7Ix4oylU7ZbPTPFugh-244GSRqMvdHsGUG6nmz2gE,17892 | |
pip/_internal/index/package_finder.py,sha256=2Rg75AOpLj8BN1jyL8EI-Iw-Hv6ibJkrYVARCht3bX8,37542 | |
pip/_internal/legacy_resolve.py,sha256=L7R72I7CjVgJlPTggmA1j4b-H8NmxNu_dKVhrpGXGps,16277 | |
pip/_internal/locations.py,sha256=VifFEqhc7FWFV8QGoEM3CpECRY8Doq7kTytytxsEgx0,6734 | |
pip/_internal/main.py,sha256=IVBnUQ-FG7DK6617uEXRB5_QJqspAsBFmTmTesYkbdQ,437 | |
pip/_internal/models/__init__.py,sha256=3DHUd_qxpPozfzouoqa9g9ts1Czr5qaHfFxbnxriepM,63 | |
pip/_internal/models/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/candidate.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/format_control.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/index.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/link.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/scheme.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/search_scope.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/selection_prefs.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/target_python.cpython-38.pyc,, | |
pip/_internal/models/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/models/candidate.py,sha256=Y58Bcm6oXUj0iS-yhmerlGo5CQJI2p0Ww9h6hR9zQDw,1150 | |
pip/_internal/models/format_control.py,sha256=ICzVjjGwfZYdX-eLLKHjMHLutEJlAGpfj09OG_eMqac,2673 | |
pip/_internal/models/index.py,sha256=K59A8-hVhBM20Xkahr4dTwP7OjkJyEqXH11UwHFVgqM,1060 | |
pip/_internal/models/link.py,sha256=y0H2ZOk0P6d1lfGUL2Pl09xFgZcRt5HwN2LElMifOpI,6827 | |
pip/_internal/models/scheme.py,sha256=vvhBrrno7eVDXcdKHiZWwxhPHf4VG5uSCEkC0QDR2RU,679 | |
pip/_internal/models/search_scope.py,sha256=2LXbU4wV8LwqdtXQXNXFYKv-IxiDI_QwSz9ZgbwtAfk,3898 | |
pip/_internal/models/selection_prefs.py,sha256=rPeif2KKjhTPXeMoQYffjqh10oWpXhdkxRDaPT1HO8k,1908 | |
pip/_internal/models/target_python.py,sha256=c-cFi6zCuo5HYbXNS3rVVpKRaHVh5yQlYEjEW23SidQ,3799 | |
pip/_internal/models/wheel.py,sha256=6KLuLKH5b0C5goWQXGSISRaq2UZtkHUEAU1y1Zsrwms,2766 | |
pip/_internal/network/__init__.py,sha256=jf6Tt5nV_7zkARBrKojIXItgejvoegVJVKUbhAa5Ioc,50 | |
pip/_internal/network/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/auth.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/cache.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/download.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/session.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/utils.cpython-38.pyc,, | |
pip/_internal/network/__pycache__/xmlrpc.cpython-38.pyc,, | |
pip/_internal/network/auth.py,sha256=K3G1ukKb3PiH8w_UnpXTz8qQsTULO-qdbfOE9zTo1fE,11119 | |
pip/_internal/network/cache.py,sha256=51CExcRkXWrgMZ7WsrZ6cmijKfViD5tVgKbBvJHO1IE,2394 | |
pip/_internal/network/download.py,sha256=3D9vdJmVwmCUMxzC-TaVI_GvVOpQna3BLEYNPCSx3Fc,6260 | |
pip/_internal/network/session.py,sha256=u1IXQfv21R1xv86ulyiB58-be4sYm90eFB0Wp8fVMYw,14702 | |
pip/_internal/network/utils.py,sha256=iiixo1OeaQ3niUWiBjg59PN6f1w7vvTww1vFriTD_IU,1959 | |
pip/_internal/network/xmlrpc.py,sha256=AL115M3vFJ8xiHVJneb8Hi0ZFeRvdPhblC89w25OG5s,1597 | |
pip/_internal/operations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 | |
pip/_internal/operations/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/operations/__pycache__/check.cpython-38.pyc,, | |
pip/_internal/operations/__pycache__/freeze.cpython-38.pyc,, | |
pip/_internal/operations/__pycache__/prepare.cpython-38.pyc,, | |
pip/_internal/operations/build/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 | |
pip/_internal/operations/build/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/operations/build/__pycache__/metadata.cpython-38.pyc,, | |
pip/_internal/operations/build/__pycache__/metadata_legacy.cpython-38.pyc,, | |
pip/_internal/operations/build/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/operations/build/__pycache__/wheel_legacy.cpython-38.pyc,, | |
pip/_internal/operations/build/metadata.py,sha256=yHMi5gHYXcXyHcvUPWHdO-UyOo3McFWljn_nHfM1O9c,1307 | |
pip/_internal/operations/build/metadata_legacy.py,sha256=4n6N7BTysqVmEpITzT2UVClyt0Peij_Im8Qm965IWB4,3957 | |
pip/_internal/operations/build/wheel.py,sha256=ntltdNP6D2Tpr4V0agssu6rE0F9LaBpJkYT6zSdhEbw,1469 | |
pip/_internal/operations/build/wheel_legacy.py,sha256=DYSxQKutwSZnmNvWkwsl2HzE2XQBxV0i0wTphjtUe90,3349 | |
pip/_internal/operations/check.py,sha256=a6uHG0daoWpmSPCdL7iYJaGQYZ-CRvPvTnCv2PnIIs0,5353 | |
pip/_internal/operations/freeze.py,sha256=td4BeRnW10EXFTZrx6VgygO3CrjqD5B9f0BGzjQm-Ew,10180 | |
pip/_internal/operations/install/__init__.py,sha256=mX7hyD2GNBO2mFGokDQ30r_GXv7Y_PLdtxcUv144e-s,51 | |
pip/_internal/operations/install/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/operations/install/__pycache__/editable_legacy.cpython-38.pyc,, | |
pip/_internal/operations/install/__pycache__/legacy.cpython-38.pyc,, | |
pip/_internal/operations/install/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/operations/install/editable_legacy.py,sha256=rJ_xs2qtDUjpY2-n6eYlVyZiNoKbOtZXZrYrcnIELt4,1488 | |
pip/_internal/operations/install/legacy.py,sha256=eBV8gHbO9sBlBc-4nuR3Sd2nikHgEcnC9khfeLiypio,4566 | |
pip/_internal/operations/install/wheel.py,sha256=xdCjH6uIUyg39Pf8tUaMFUN4a7eozJAFMb_wKcgQlsY,23012 | |
pip/_internal/operations/prepare.py,sha256=ro2teBlbBpkRJhBKraP9CoJgVLpueSk62ziWhRToXww,20942 | |
pip/_internal/pep425tags.py,sha256=SlIQokevkoKnXhoK3PZvXiDoj8hFKoJ7thDifDtga3k,5490 | |
pip/_internal/pyproject.py,sha256=kB966ZCSxiZQRa3W2RXN9as5pRuKW6Elnb4xdqDxASg,7404 | |
pip/_internal/req/__init__.py,sha256=UVaYPlHZVGRBQQPjvGC_6jJDQtewXm0ws-8Lxhg_TiY,2671 | |
pip/_internal/req/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/constructors.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/req_file.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/req_install.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/req_set.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/req_tracker.cpython-38.pyc,, | |
pip/_internal/req/__pycache__/req_uninstall.cpython-38.pyc,, | |
pip/_internal/req/constructors.py,sha256=w5-kWWVCqlSqcIBitw86yq7XGMPpKrHDfQZSE2mJ_xc,14388 | |
pip/_internal/req/req_file.py,sha256=ECqRUicCw5Y08R1YynZAAp8dSKQhDXoc1Q-mY3a9b6I,18485 | |
pip/_internal/req/req_install.py,sha256=wjsIr4lDpbVSLqANKJI9mXwRVHaRxcnj8q30UiHoLRA,30442 | |
pip/_internal/req/req_set.py,sha256=GsrKmupRKhNMhjkofVfCEHEHfgEvYBxClaQH5xLBQHg,8066 | |
pip/_internal/req/req_tracker.py,sha256=27fvVG8Y2MJS1KpU2rBMnQyUEMHG4lkHT_bzbzQK-c0,4723 | |
pip/_internal/req/req_uninstall.py,sha256=DWnOsuyYGju6-sylyoCm7GtUNevn9qMAVhjAGLcdXUE,23609 | |
pip/_internal/self_outdated_check.py,sha256=3KO1pTJUuYaiV9X0t87I9PimkGL82HbhLWbocqKZpBU,8009 | |
pip/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 | |
pip/_internal/utils/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/appdirs.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/compat.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/deprecation.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/distutils_args.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/encoding.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/entrypoints.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/filesystem.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/filetypes.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/glibc.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/hashes.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/inject_securetransport.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/logging.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/marker_files.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/misc.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/models.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/packaging.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/pkg_resources.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/setuptools_build.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/subprocess.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/temp_dir.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/typing.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/ui.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/unpacking.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/urls.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/virtualenv.cpython-38.pyc,, | |
pip/_internal/utils/__pycache__/wheel.cpython-38.pyc,, | |
pip/_internal/utils/appdirs.py,sha256=frpKbfJiyKLgpPDYNDrPtkfaZ0akY9SyB7ryPV29sMg,1144 | |
pip/_internal/utils/compat.py,sha256=D7FKGLBdQwWH-dHIGaoWMawDZWBYApvtJVL1kFPJ930,8869 | |
pip/_internal/utils/deprecation.py,sha256=pBnNogoA4UGTxa_JDnPXBRRYpKMbExAhXpBwAwklOBs,3318 | |
pip/_internal/utils/distutils_args.py,sha256=a56mblNxk9BGifbpEETG61mmBrqhjtjRkJ4HYn-oOEE,1350 | |
pip/_internal/utils/encoding.py,sha256=hxZz0t3Whw3d4MHQEiofxalTlfKwxFdLc8fpeGfhKo8,1320 | |
pip/_internal/utils/entrypoints.py,sha256=vHcNpnksCv6mllihU6hfifdsKPEjwcaJ1aLIXEaynaU,1152 | |
pip/_internal/utils/filesystem.py,sha256=PXa3vMcz4mbEKtkD0joFI8pBwddLQxhfPFOkVH5xjfE,5255 | |
pip/_internal/utils/filetypes.py,sha256=R2FwzoeX7b-rZALOXx5cuO8VPPMhUQ4ne7wm3n3IcWA,571 | |
pip/_internal/utils/glibc.py,sha256=LOeNGgawCKS-4ke9fii78fwXD73dtNav3uxz1Bf-Ab8,3297 | |
pip/_internal/utils/hashes.py,sha256=my-wSnAWEDvl_8rQaOQcVIWjwh1-f_QiEvGy9TPf53U,3942 | |
pip/_internal/utils/inject_securetransport.py,sha256=M17ZlFVY66ApgeASVjKKLKNz0LAfk-SyU0HZ4ZB6MmI,810 | |
pip/_internal/utils/logging.py,sha256=aJL7NldPhS5KGFof6Qt3o3MG5cjm5TOoo7bGRu9_wsg,13033 | |
pip/_internal/utils/marker_files.py,sha256=CO5djQlrPIozJpJybViH_insoAaBGY1aqEt6-cC-iW0,741 | |
pip/_internal/utils/misc.py,sha256=uIb58Hiu_g2HRORo2aMcgnW_7R5d-5wUAuoW0fA2ZME,26085 | |
pip/_internal/utils/models.py,sha256=IA0hw_T4awQzui0kqfIEASm5yLtgZAB08ag59Nip5G8,1148 | |
pip/_internal/utils/packaging.py,sha256=VtiwcAAL7LBi7tGL2je7LeW4bE11KMHGCsJ1NZY5XtM,3035 | |
pip/_internal/utils/pkg_resources.py,sha256=ZX-k7V5q_aNWyDse92nN7orN1aCpRLsaxzpkBZ1XKzU,1254 | |
pip/_internal/utils/setuptools_build.py,sha256=DouaVolV9olDDFIIN9IszaL-FHdNaZt10ufOZFH9ZAU,5070 | |
pip/_internal/utils/subprocess.py,sha256=Ph3x5eHQBxFotyGhpZN8asSMBud-BBkmgaNfARG-di8,9922 | |
pip/_internal/utils/temp_dir.py,sha256=87Ib8aNic_hoSDEmUYJHTQIn5-prL2AYL5u_yZ3s4sI,7768 | |
pip/_internal/utils/typing.py,sha256=xkYwOeHlf4zsHXBDC4310HtEqwhQcYXFPq2h35Tcrl0,1401 | |
pip/_internal/utils/ui.py,sha256=0FNxXlGtbpPtTviv2oXS9t8bQG_NBdfUgP4GbubhS9U,13911 | |
pip/_internal/utils/unpacking.py,sha256=M944JTSiapBOSKLWu7lbawpVHSE7flfzZTEr3TAG7v8,9438 | |
pip/_internal/utils/urls.py,sha256=aNV9wq5ClUmrz6sG-al7hEWJ4ToitOy7l82CmFGFNW8,1481 | |
pip/_internal/utils/virtualenv.py,sha256=Q3S1WPlI7JWpGOT2jUVJ8l2chm_k7VPJ9cHA_cUluEU,3396 | |
pip/_internal/utils/wheel.py,sha256=grTRwZtMQwApwbbSPmRVLtac6FKy6SVKeCXNkWyyePA,7302 | |
pip/_internal/vcs/__init__.py,sha256=viJxJRqRE_mVScum85bgQIXAd6o0ozFt18VpC-qIJrM,617 | |
pip/_internal/vcs/__pycache__/__init__.cpython-38.pyc,, | |
pip/_internal/vcs/__pycache__/bazaar.cpython-38.pyc,, | |
pip/_internal/vcs/__pycache__/git.cpython-38.pyc,, | |
pip/_internal/vcs/__pycache__/mercurial.cpython-38.pyc,, | |
pip/_internal/vcs/__pycache__/subversion.cpython-38.pyc,, | |
pip/_internal/vcs/__pycache__/versioncontrol.cpython-38.pyc,, | |
pip/_internal/vcs/bazaar.py,sha256=84q1-kj1_nJ9AMzMu8RmMp-riRZu81M7K9kowcYgi3U,3957 | |
pip/_internal/vcs/git.py,sha256=X0j5jv_x3ZnM_NP09B1ZDxW-PAmfHzqOqX7Wf5XW--0,14058 | |
pip/_internal/vcs/mercurial.py,sha256=2mg7BdYI_Fe00fF6omaNccFQLPHBsDBG5CAEzvqn5sA,5110 | |
pip/_internal/vcs/subversion.py,sha256=Fpwy71AmuqXnoKi6h1SrXRtPjEMn8fieuM1O4j01IBg,12292 | |
pip/_internal/vcs/versioncontrol.py,sha256=nqoaM1_rzx24WnHtihXA8RcPpnUae0sV2sR_LS_5HFA,22600 | |
pip/_internal/wheel_builder.py,sha256=gr9jE14W5ZuYblpldo-tpRuyG0e0AVmHLttImuAvXlE,9441 | |
pip/_vendor/__init__.py,sha256=v3ZUr2bBNfKRfNOPnrjq2aU7ZjgGiamksWvYlJM5nco,4894 | |
pip/_vendor/__pycache__/__init__.cpython-38.pyc,, |
pip |
Wheel-Version: 1.0 | |
Generator: bdist_wheel (0.34.2) | |
Root-Is-Purelib: true | |
Tag: py2-none-any | |
Tag: py3-none-any | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional | |
__version__ = "20.0.2" | |
def main(args=None): | |
# type: (Optional[List[str]]) -> int | |
"""This is an internal API only meant for use by pip's own console scripts. | |
For additional details, see https://github.com/pypa/pip/issues/7498. | |
""" | |
from pip._internal.utils.entrypoints import _wrapper | |
return _wrapper(args) |
from __future__ import absolute_import | |
import os | |
import sys | |
# If we are running from a wheel, add the wheel to sys.path | |
# This allows the usage python pip-*.whl/pip install pip-*.whl | |
if __package__ == '': | |
# __file__ is pip-*.whl/pip/__main__.py | |
# first dirname call strips of '/__main__.py', second strips off '/pip' | |
# Resulting path is the name of the wheel itself | |
# Add that to sys.path so we can import pip | |
path = os.path.dirname(os.path.dirname(__file__)) | |
sys.path.insert(0, path) | |
from pip._internal.cli.main import main as _main # isort:skip # noqa | |
if __name__ == '__main__': | |
sys.exit(_main()) |
#!/usr/bin/env python | |
import pip._internal.utils.inject_securetransport # noqa | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, List | |
def main(args=None): | |
# type: (Optional[List[str]]) -> int | |
"""This is preserved for old console scripts that may still be referencing | |
it. | |
For additional details, see https://github.com/pypa/pip/issues/7498. | |
""" | |
from pip._internal.utils.entrypoints import _wrapper | |
return _wrapper(args) |
"""Build Environment used for isolation during sdist building | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
import logging | |
import os | |
import sys | |
import textwrap | |
from collections import OrderedDict | |
from distutils.sysconfig import get_python_lib | |
from sysconfig import get_paths | |
from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet | |
from pip import __file__ as pip_location | |
from pip._internal.utils.subprocess import call_subprocess | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.ui import open_spinner | |
if MYPY_CHECK_RUNNING: | |
from typing import Tuple, Set, Iterable, Optional, List | |
from pip._internal.index.package_finder import PackageFinder | |
logger = logging.getLogger(__name__) | |
class _Prefix: | |
def __init__(self, path): | |
# type: (str) -> None | |
self.path = path | |
self.setup = False | |
self.bin_dir = get_paths( | |
'nt' if os.name == 'nt' else 'posix_prefix', | |
vars={'base': path, 'platbase': path} | |
)['scripts'] | |
# Note: prefer distutils' sysconfig to get the | |
# library paths so PyPy is correctly supported. | |
purelib = get_python_lib(plat_specific=False, prefix=path) | |
platlib = get_python_lib(plat_specific=True, prefix=path) | |
if purelib == platlib: | |
self.lib_dirs = [purelib] | |
else: | |
self.lib_dirs = [purelib, platlib] | |
class BuildEnvironment(object): | |
"""Creates and manages an isolated environment to install build deps | |
""" | |
def __init__(self): | |
# type: () -> None | |
self._temp_dir = TempDirectory(kind="build-env") | |
self._prefixes = OrderedDict(( | |
(name, _Prefix(os.path.join(self._temp_dir.path, name))) | |
for name in ('normal', 'overlay') | |
)) | |
self._bin_dirs = [] # type: List[str] | |
self._lib_dirs = [] # type: List[str] | |
for prefix in reversed(list(self._prefixes.values())): | |
self._bin_dirs.append(prefix.bin_dir) | |
self._lib_dirs.extend(prefix.lib_dirs) | |
# Customize site to: | |
# - ensure .pth files are honored | |
# - prevent access to system site packages | |
system_sites = { | |
os.path.normcase(site) for site in ( | |
get_python_lib(plat_specific=False), | |
get_python_lib(plat_specific=True), | |
) | |
} | |
self._site_dir = os.path.join(self._temp_dir.path, 'site') | |
if not os.path.exists(self._site_dir): | |
os.mkdir(self._site_dir) | |
with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: | |
fp.write(textwrap.dedent( | |
''' | |
import os, site, sys | |
# First, drop system-sites related paths. | |
original_sys_path = sys.path[:] | |
known_paths = set() | |
for path in {system_sites!r}: | |
site.addsitedir(path, known_paths=known_paths) | |
system_paths = set( | |
os.path.normcase(path) | |
for path in sys.path[len(original_sys_path):] | |
) | |
original_sys_path = [ | |
path for path in original_sys_path | |
if os.path.normcase(path) not in system_paths | |
] | |
sys.path = original_sys_path | |
# Second, add lib directories. | |
# ensuring .pth file are processed. | |
for path in {lib_dirs!r}: | |
assert not path in sys.path | |
site.addsitedir(path) | |
''' | |
).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) | |
def __enter__(self): | |
self._save_env = { | |
name: os.environ.get(name, None) | |
for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') | |
} | |
path = self._bin_dirs[:] | |
old_path = self._save_env['PATH'] | |
if old_path: | |
path.extend(old_path.split(os.pathsep)) | |
pythonpath = [self._site_dir] | |
os.environ.update({ | |
'PATH': os.pathsep.join(path), | |
'PYTHONNOUSERSITE': '1', | |
'PYTHONPATH': os.pathsep.join(pythonpath), | |
}) | |
def __exit__(self, exc_type, exc_val, exc_tb): | |
for varname, old_value in self._save_env.items(): | |
if old_value is None: | |
os.environ.pop(varname, None) | |
else: | |
os.environ[varname] = old_value | |
def cleanup(self): | |
# type: () -> None | |
self._temp_dir.cleanup() | |
def check_requirements(self, reqs): | |
# type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] | |
"""Return 2 sets: | |
- conflicting requirements: set of (installed, wanted) reqs tuples | |
- missing requirements: set of reqs | |
""" | |
missing = set() | |
conflicting = set() | |
if reqs: | |
ws = WorkingSet(self._lib_dirs) | |
for req in reqs: | |
try: | |
if ws.find(Requirement.parse(req)) is None: | |
missing.add(req) | |
except VersionConflict as e: | |
conflicting.add((str(e.args[0].as_requirement()), | |
str(e.args[1]))) | |
return conflicting, missing | |
def install_requirements( | |
self, | |
finder, # type: PackageFinder | |
requirements, # type: Iterable[str] | |
prefix_as_string, # type: str | |
message # type: Optional[str] | |
): | |
# type: (...) -> None | |
prefix = self._prefixes[prefix_as_string] | |
assert not prefix.setup | |
prefix.setup = True | |
if not requirements: | |
return | |
args = [ | |
sys.executable, os.path.dirname(pip_location), 'install', | |
'--ignore-installed', '--no-user', '--prefix', prefix.path, | |
'--no-warn-script-location', | |
] # type: List[str] | |
if logger.getEffectiveLevel() <= logging.DEBUG: | |
args.append('-v') | |
for format_control in ('no_binary', 'only_binary'): | |
formats = getattr(finder.format_control, format_control) | |
args.extend(('--' + format_control.replace('_', '-'), | |
','.join(sorted(formats or {':none:'})))) | |
index_urls = finder.index_urls | |
if index_urls: | |
args.extend(['-i', index_urls[0]]) | |
for extra_index in index_urls[1:]: | |
args.extend(['--extra-index-url', extra_index]) | |
else: | |
args.append('--no-index') | |
for link in finder.find_links: | |
args.extend(['--find-links', link]) | |
for host in finder.trusted_hosts: | |
args.extend(['--trusted-host', host]) | |
if finder.allow_all_prereleases: | |
args.append('--pre') | |
args.append('--') | |
args.extend(requirements) | |
with open_spinner(message) as spinner: | |
call_subprocess(args, spinner=spinner) | |
class NoOpBuildEnvironment(BuildEnvironment): | |
"""A no-op drop-in replacement for BuildEnvironment | |
""" | |
def __init__(self): | |
pass | |
def __enter__(self): | |
pass | |
def __exit__(self, exc_type, exc_val, exc_tb): | |
pass | |
def cleanup(self): | |
pass | |
def install_requirements(self, finder, requirements, prefix, message): | |
raise NotImplementedError() |
"""Cache Management | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import hashlib | |
import json | |
import logging | |
import os | |
from pip._vendor.packaging.tags import interpreter_name, interpreter_version | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal.exceptions import InvalidWheelFilename | |
from pip._internal.models.link import Link | |
from pip._internal.models.wheel import Wheel | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Set, List, Any, Dict | |
from pip._vendor.packaging.tags import Tag | |
from pip._internal.models.format_control import FormatControl | |
logger = logging.getLogger(__name__) | |
def _hash_dict(d): | |
# type: (Dict[str, str]) -> str | |
"""Return a stable sha224 of a dictionary.""" | |
s = json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True) | |
return hashlib.sha224(s.encode("ascii")).hexdigest() | |
class Cache(object): | |
"""An abstract class - provides cache directories for data from links | |
:param cache_dir: The root of the cache. | |
:param format_control: An object of FormatControl class to limit | |
binaries being read from the cache. | |
:param allowed_formats: which formats of files the cache should store. | |
('binary' and 'source' are the only allowed values) | |
""" | |
def __init__(self, cache_dir, format_control, allowed_formats): | |
# type: (str, FormatControl, Set[str]) -> None | |
super(Cache, self).__init__() | |
assert not cache_dir or os.path.isabs(cache_dir) | |
self.cache_dir = cache_dir or None | |
self.format_control = format_control | |
self.allowed_formats = allowed_formats | |
_valid_formats = {"source", "binary"} | |
assert self.allowed_formats.union(_valid_formats) == _valid_formats | |
def _get_cache_path_parts_legacy(self, link): | |
# type: (Link) -> List[str] | |
"""Get parts of part that must be os.path.joined with cache_dir | |
Legacy cache key (pip < 20) for compatibility with older caches. | |
""" | |
# We want to generate an url to use as our cache key, we don't want to | |
# just re-use the URL because it might have other items in the fragment | |
# and we don't care about those. | |
key_parts = [link.url_without_fragment] | |
if link.hash_name is not None and link.hash is not None: | |
key_parts.append("=".join([link.hash_name, link.hash])) | |
key_url = "#".join(key_parts) | |
# Encode our key url with sha224, we'll use this because it has similar | |
# security properties to sha256, but with a shorter total output (and | |
# thus less secure). However the differences don't make a lot of | |
# difference for our use case here. | |
hashed = hashlib.sha224(key_url.encode()).hexdigest() | |
# We want to nest the directories some to prevent having a ton of top | |
# level directories where we might run out of sub directories on some | |
# FS. | |
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] | |
return parts | |
def _get_cache_path_parts(self, link): | |
# type: (Link) -> List[str] | |
"""Get parts of part that must be os.path.joined with cache_dir | |
""" | |
# We want to generate an url to use as our cache key, we don't want to | |
# just re-use the URL because it might have other items in the fragment | |
# and we don't care about those. | |
key_parts = {"url": link.url_without_fragment} | |
if link.hash_name is not None and link.hash is not None: | |
key_parts[link.hash_name] = link.hash | |
if link.subdirectory_fragment: | |
key_parts["subdirectory"] = link.subdirectory_fragment | |
# Include interpreter name, major and minor version in cache key | |
# to cope with ill-behaved sdists that build a different wheel | |
# depending on the python version their setup.py is being run on, | |
# and don't encode the difference in compatibility tags. | |
# https://github.com/pypa/pip/issues/7296 | |
key_parts["interpreter_name"] = interpreter_name() | |
key_parts["interpreter_version"] = interpreter_version() | |
# Encode our key url with sha224, we'll use this because it has similar | |
# security properties to sha256, but with a shorter total output (and | |
# thus less secure). However the differences don't make a lot of | |
# difference for our use case here. | |
hashed = _hash_dict(key_parts) | |
# We want to nest the directories some to prevent having a ton of top | |
# level directories where we might run out of sub directories on some | |
# FS. | |
parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] | |
return parts | |
def _get_candidates(self, link, canonical_package_name): | |
# type: (Link, Optional[str]) -> List[Any] | |
can_not_cache = ( | |
not self.cache_dir or | |
not canonical_package_name or | |
not link | |
) | |
if can_not_cache: | |
return [] | |
formats = self.format_control.get_allowed_formats( | |
canonical_package_name | |
) | |
if not self.allowed_formats.intersection(formats): | |
return [] | |
candidates = [] | |
path = self.get_path_for_link(link) | |
if os.path.isdir(path): | |
for candidate in os.listdir(path): | |
candidates.append((candidate, path)) | |
# TODO remove legacy path lookup in pip>=21 | |
legacy_path = self.get_path_for_link_legacy(link) | |
if os.path.isdir(legacy_path): | |
for candidate in os.listdir(legacy_path): | |
candidates.append((candidate, legacy_path)) | |
return candidates | |
def get_path_for_link_legacy(self, link): | |
# type: (Link) -> str | |
raise NotImplementedError() | |
def get_path_for_link(self, link): | |
# type: (Link) -> str | |
"""Return a directory to store cached items in for link. | |
""" | |
raise NotImplementedError() | |
def get( | |
self, | |
link, # type: Link | |
package_name, # type: Optional[str] | |
supported_tags, # type: List[Tag] | |
): | |
# type: (...) -> Link | |
"""Returns a link to a cached item if it exists, otherwise returns the | |
passed link. | |
""" | |
raise NotImplementedError() | |
def cleanup(self): | |
# type: () -> None | |
pass | |
class SimpleWheelCache(Cache): | |
"""A cache of wheels for future installs. | |
""" | |
def __init__(self, cache_dir, format_control): | |
# type: (str, FormatControl) -> None | |
super(SimpleWheelCache, self).__init__( | |
cache_dir, format_control, {"binary"} | |
) | |
def get_path_for_link_legacy(self, link): | |
# type: (Link) -> str | |
parts = self._get_cache_path_parts_legacy(link) | |
return os.path.join(self.cache_dir, "wheels", *parts) | |
def get_path_for_link(self, link): | |
# type: (Link) -> str | |
"""Return a directory to store cached wheels for link | |
Because there are M wheels for any one sdist, we provide a directory | |
to cache them in, and then consult that directory when looking up | |
cache hits. | |
We only insert things into the cache if they have plausible version | |
numbers, so that we don't contaminate the cache with things that were | |
not unique. E.g. ./package might have dozens of installs done for it | |
and build a version of 0.0...and if we built and cached a wheel, we'd | |
end up using the same wheel even if the source has been edited. | |
:param link: The link of the sdist for which this will cache wheels. | |
""" | |
parts = self._get_cache_path_parts(link) | |
# Store wheels within the root cache_dir | |
return os.path.join(self.cache_dir, "wheels", *parts) | |
def get( | |
self, | |
link, # type: Link | |
package_name, # type: Optional[str] | |
supported_tags, # type: List[Tag] | |
): | |
# type: (...) -> Link | |
candidates = [] | |
if not package_name: | |
return link | |
canonical_package_name = canonicalize_name(package_name) | |
for wheel_name, wheel_dir in self._get_candidates( | |
link, canonical_package_name | |
): | |
try: | |
wheel = Wheel(wheel_name) | |
except InvalidWheelFilename: | |
continue | |
if canonicalize_name(wheel.name) != canonical_package_name: | |
logger.debug( | |
"Ignoring cached wheel {} for {} as it " | |
"does not match the expected distribution name {}.".format( | |
wheel_name, link, package_name | |
) | |
) | |
continue | |
if not wheel.supported(supported_tags): | |
# Built for a different python/arch/etc | |
continue | |
candidates.append( | |
( | |
wheel.support_index_min(supported_tags), | |
wheel_name, | |
wheel_dir, | |
) | |
) | |
if not candidates: | |
return link | |
_, wheel_name, wheel_dir = min(candidates) | |
return Link(path_to_url(os.path.join(wheel_dir, wheel_name))) | |
class EphemWheelCache(SimpleWheelCache): | |
"""A SimpleWheelCache that creates it's own temporary cache directory | |
""" | |
def __init__(self, format_control): | |
# type: (FormatControl) -> None | |
self._temp_dir = TempDirectory(kind="ephem-wheel-cache") | |
super(EphemWheelCache, self).__init__( | |
self._temp_dir.path, format_control | |
) | |
def cleanup(self): | |
# type: () -> None | |
self._temp_dir.cleanup() | |
class WheelCache(Cache): | |
"""Wraps EphemWheelCache and SimpleWheelCache into a single Cache | |
This Cache allows for gracefully degradation, using the ephem wheel cache | |
when a certain link is not found in the simple wheel cache first. | |
""" | |
def __init__(self, cache_dir, format_control): | |
# type: (str, FormatControl) -> None | |
super(WheelCache, self).__init__( | |
cache_dir, format_control, {'binary'} | |
) | |
self._wheel_cache = SimpleWheelCache(cache_dir, format_control) | |
self._ephem_cache = EphemWheelCache(format_control) | |
def get_path_for_link_legacy(self, link): | |
# type: (Link) -> str | |
return self._wheel_cache.get_path_for_link_legacy(link) | |
def get_path_for_link(self, link): | |
# type: (Link) -> str | |
return self._wheel_cache.get_path_for_link(link) | |
def get_ephem_path_for_link(self, link): | |
# type: (Link) -> str | |
return self._ephem_cache.get_path_for_link(link) | |
def get( | |
self, | |
link, # type: Link | |
package_name, # type: Optional[str] | |
supported_tags, # type: List[Tag] | |
): | |
# type: (...) -> Link | |
retval = self._wheel_cache.get( | |
link=link, | |
package_name=package_name, | |
supported_tags=supported_tags, | |
) | |
if retval is not link: | |
return retval | |
return self._ephem_cache.get( | |
link=link, | |
package_name=package_name, | |
supported_tags=supported_tags, | |
) | |
def cleanup(self): | |
# type: () -> None | |
self._wheel_cache.cleanup() | |
self._ephem_cache.cleanup() |
"""Subpackage containing all of pip's command line interface related code | |
""" | |
# This file intentionally does not import submodules |
"""Logic that powers autocompletion installed by ``pip completion``. | |
""" | |
import optparse | |
import os | |
import sys | |
from itertools import chain | |
from pip._internal.cli.main_parser import create_main_parser | |
from pip._internal.commands import commands_dict, create_command | |
from pip._internal.utils.misc import get_installed_distributions | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Iterable, List, Optional | |
def autocomplete(): | |
# type: () -> None | |
"""Entry Point for completion of main and subcommand options. | |
""" | |
# Don't complete if user hasn't sourced bash_completion file. | |
if 'PIP_AUTO_COMPLETE' not in os.environ: | |
return | |
cwords = os.environ['COMP_WORDS'].split()[1:] | |
cword = int(os.environ['COMP_CWORD']) | |
try: | |
current = cwords[cword - 1] | |
except IndexError: | |
current = '' | |
parser = create_main_parser() | |
subcommands = list(commands_dict) | |
options = [] | |
# subcommand | |
subcommand_name = None # type: Optional[str] | |
for word in cwords: | |
if word in subcommands: | |
subcommand_name = word | |
break | |
# subcommand options | |
if subcommand_name is not None: | |
# special case: 'help' subcommand has no options | |
if subcommand_name == 'help': | |
sys.exit(1) | |
# special case: list locally installed dists for show and uninstall | |
should_list_installed = ( | |
subcommand_name in ['show', 'uninstall'] and | |
not current.startswith('-') | |
) | |
if should_list_installed: | |
installed = [] | |
lc = current.lower() | |
for dist in get_installed_distributions(local_only=True): | |
if dist.key.startswith(lc) and dist.key not in cwords[1:]: | |
installed.append(dist.key) | |
# if there are no dists installed, fall back to option completion | |
if installed: | |
for dist in installed: | |
print(dist) | |
sys.exit(1) | |
subcommand = create_command(subcommand_name) | |
for opt in subcommand.parser.option_list_all: | |
if opt.help != optparse.SUPPRESS_HELP: | |
for opt_str in opt._long_opts + opt._short_opts: | |
options.append((opt_str, opt.nargs)) | |
# filter out previously specified options from available options | |
prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] | |
options = [(x, v) for (x, v) in options if x not in prev_opts] | |
# filter options by current input | |
options = [(k, v) for k, v in options if k.startswith(current)] | |
# get completion type given cwords and available subcommand options | |
completion_type = get_path_completion_type( | |
cwords, cword, subcommand.parser.option_list_all, | |
) | |
# get completion files and directories if ``completion_type`` is | |
# ``<file>``, ``<dir>`` or ``<path>`` | |
if completion_type: | |
paths = auto_complete_paths(current, completion_type) | |
options = [(path, 0) for path in paths] | |
for option in options: | |
opt_label = option[0] | |
# append '=' to options which require args | |
if option[1] and option[0][:2] == "--": | |
opt_label += '=' | |
print(opt_label) | |
else: | |
# show main parser options only when necessary | |
opts = [i.option_list for i in parser.option_groups] | |
opts.append(parser.option_list) | |
flattened_opts = chain.from_iterable(opts) | |
if current.startswith('-'): | |
for opt in flattened_opts: | |
if opt.help != optparse.SUPPRESS_HELP: | |
subcommands += opt._long_opts + opt._short_opts | |
else: | |
# get completion type given cwords and all available options | |
completion_type = get_path_completion_type(cwords, cword, | |
flattened_opts) | |
if completion_type: | |
subcommands = list(auto_complete_paths(current, | |
completion_type)) | |
print(' '.join([x for x in subcommands if x.startswith(current)])) | |
sys.exit(1) | |
def get_path_completion_type(cwords, cword, opts): | |
# type: (List[str], int, Iterable[Any]) -> Optional[str] | |
"""Get the type of path completion (``file``, ``dir``, ``path`` or None) | |
:param cwords: same as the environmental variable ``COMP_WORDS`` | |
:param cword: same as the environmental variable ``COMP_CWORD`` | |
:param opts: The available options to check | |
:return: path completion type (``file``, ``dir``, ``path`` or None) | |
""" | |
if cword < 2 or not cwords[cword - 2].startswith('-'): | |
return None | |
for opt in opts: | |
if opt.help == optparse.SUPPRESS_HELP: | |
continue | |
for o in str(opt).split('/'): | |
if cwords[cword - 2].split('=')[0] == o: | |
if not opt.metavar or any( | |
x in ('path', 'file', 'dir') | |
for x in opt.metavar.split('/')): | |
return opt.metavar | |
return None | |
def auto_complete_paths(current, completion_type): | |
# type: (str, str) -> Iterable[str] | |
"""If ``completion_type`` is ``file`` or ``path``, list all regular files | |
and directories starting with ``current``; otherwise only list directories | |
starting with ``current``. | |
:param current: The word to be completed | |
:param completion_type: path completion type(`file`, `path` or `dir`)i | |
:return: A generator of regular files and/or directories | |
""" | |
directory, filename = os.path.split(current) | |
current_path = os.path.abspath(directory) | |
# Don't complete paths if they can't be accessed | |
if not os.access(current_path, os.R_OK): | |
return | |
filename = os.path.normcase(filename) | |
# list all files that start with ``filename`` | |
file_list = (x for x in os.listdir(current_path) | |
if os.path.normcase(x).startswith(filename)) | |
for f in file_list: | |
opt = os.path.join(current_path, f) | |
comp_file = os.path.normcase(os.path.join(directory, f)) | |
# complete regular files when there is not ``<dir>`` after option | |
# complete directories when there is ``<file>``, ``<path>`` or | |
# ``<dir>``after option | |
if completion_type != 'dir' and os.path.isfile(opt): | |
yield comp_file | |
elif os.path.isdir(opt): | |
yield os.path.join(comp_file, '') |
"""Base Command class, and related routines""" | |
from __future__ import absolute_import, print_function | |
import logging | |
import logging.config | |
import optparse | |
import os | |
import platform | |
import sys | |
import traceback | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.command_context import CommandContextMixIn | |
from pip._internal.cli.parser import ( | |
ConfigOptionParser, | |
UpdatingDefaultsHelpFormatter, | |
) | |
from pip._internal.cli.status_codes import ( | |
ERROR, | |
PREVIOUS_BUILD_DIR_ERROR, | |
SUCCESS, | |
UNKNOWN_ERROR, | |
VIRTUALENV_NOT_FOUND, | |
) | |
from pip._internal.exceptions import ( | |
BadCommand, | |
CommandError, | |
InstallationError, | |
PreviousBuildDirError, | |
UninstallationError, | |
) | |
from pip._internal.utils.deprecation import deprecated | |
from pip._internal.utils.filesystem import check_path_owner | |
from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging | |
from pip._internal.utils.misc import get_prog, normalize_path | |
from pip._internal.utils.temp_dir import global_tempdir_manager | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.virtualenv import running_under_virtualenv | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Tuple, Any | |
from optparse import Values | |
__all__ = ['Command'] | |
logger = logging.getLogger(__name__) | |
class Command(CommandContextMixIn): | |
usage = None # type: str | |
ignore_require_venv = False # type: bool | |
def __init__(self, name, summary, isolated=False): | |
# type: (str, str, bool) -> None | |
super(Command, self).__init__() | |
parser_kw = { | |
'usage': self.usage, | |
'prog': '%s %s' % (get_prog(), name), | |
'formatter': UpdatingDefaultsHelpFormatter(), | |
'add_help_option': False, | |
'name': name, | |
'description': self.__doc__, | |
'isolated': isolated, | |
} | |
self.name = name | |
self.summary = summary | |
self.parser = ConfigOptionParser(**parser_kw) | |
# Commands should add options to this option group | |
optgroup_name = '%s Options' % self.name.capitalize() | |
self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name) | |
# Add the general options | |
gen_opts = cmdoptions.make_option_group( | |
cmdoptions.general_group, | |
self.parser, | |
) | |
self.parser.add_option_group(gen_opts) | |
def handle_pip_version_check(self, options): | |
# type: (Values) -> None | |
""" | |
This is a no-op so that commands by default do not do the pip version | |
check. | |
""" | |
# Make sure we do the pip version check if the index_group options | |
# are present. | |
assert not hasattr(options, 'no_index') | |
def run(self, options, args): | |
# type: (Values, List[Any]) -> Any | |
raise NotImplementedError | |
def parse_args(self, args): | |
# type: (List[str]) -> Tuple[Any, Any] | |
# factored out for testability | |
return self.parser.parse_args(args) | |
def main(self, args): | |
# type: (List[str]) -> int | |
try: | |
with self.main_context(): | |
return self._main(args) | |
finally: | |
logging.shutdown() | |
def _main(self, args): | |
# type: (List[str]) -> int | |
# Intentionally set as early as possible so globally-managed temporary | |
# directories are available to the rest of the code. | |
self.enter_context(global_tempdir_manager()) | |
options, args = self.parse_args(args) | |
# Set verbosity so that it can be used elsewhere. | |
self.verbosity = options.verbose - options.quiet | |
level_number = setup_logging( | |
verbosity=self.verbosity, | |
no_color=options.no_color, | |
user_log_file=options.log, | |
) | |
if ( | |
sys.version_info[:2] == (2, 7) and | |
not options.no_python_version_warning | |
): | |
message = ( | |
"A future version of pip will drop support for Python 2.7. " | |
"More details about Python 2 support in pip, can be found at " | |
"https://pip.pypa.io/en/latest/development/release-process/#python-2-support" # noqa | |
) | |
if platform.python_implementation() == "CPython": | |
message = ( | |
"Python 2.7 reached the end of its life on January " | |
"1st, 2020. Please upgrade your Python as Python 2.7 " | |
"is no longer maintained. " | |
) + message | |
deprecated(message, replacement=None, gone_in=None) | |
if options.skip_requirements_regex: | |
deprecated( | |
"--skip-requirements-regex is unsupported and will be removed", | |
replacement=( | |
"manage requirements/constraints files explicitly, " | |
"possibly generating them from metadata" | |
), | |
gone_in="20.1", | |
issue=7297, | |
) | |
# TODO: Try to get these passing down from the command? | |
# without resorting to os.environ to hold these. | |
# This also affects isolated builds and it should. | |
if options.no_input: | |
os.environ['PIP_NO_INPUT'] = '1' | |
if options.exists_action: | |
os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) | |
if options.require_venv and not self.ignore_require_venv: | |
# If a venv is required check if it can really be found | |
if not running_under_virtualenv(): | |
logger.critical( | |
'Could not find an activated virtualenv (required).' | |
) | |
sys.exit(VIRTUALENV_NOT_FOUND) | |
if options.cache_dir: | |
options.cache_dir = normalize_path(options.cache_dir) | |
if not check_path_owner(options.cache_dir): | |
logger.warning( | |
"The directory '%s' or its parent directory is not owned " | |
"or is not writable by the current user. The cache " | |
"has been disabled. Check the permissions and owner of " | |
"that directory. If executing pip with sudo, you may want " | |
"sudo's -H flag.", | |
options.cache_dir, | |
) | |
options.cache_dir = None | |
try: | |
status = self.run(options, args) | |
# FIXME: all commands should return an exit status | |
# and when it is done, isinstance is not needed anymore | |
if isinstance(status, int): | |
return status | |
except PreviousBuildDirError as exc: | |
logger.critical(str(exc)) | |
logger.debug('Exception information:', exc_info=True) | |
return PREVIOUS_BUILD_DIR_ERROR | |
except (InstallationError, UninstallationError, BadCommand) as exc: | |
logger.critical(str(exc)) | |
logger.debug('Exception information:', exc_info=True) | |
return ERROR | |
except CommandError as exc: | |
logger.critical('%s', exc) | |
logger.debug('Exception information:', exc_info=True) | |
return ERROR | |
except BrokenStdoutLoggingError: | |
# Bypass our logger and write any remaining messages to stderr | |
# because stdout no longer works. | |
print('ERROR: Pipe to stdout was broken', file=sys.stderr) | |
if level_number <= logging.DEBUG: | |
traceback.print_exc(file=sys.stderr) | |
return ERROR | |
except KeyboardInterrupt: | |
logger.critical('Operation cancelled by user') | |
logger.debug('Exception information:', exc_info=True) | |
return ERROR | |
except BaseException: | |
logger.critical('Exception:', exc_info=True) | |
return UNKNOWN_ERROR | |
finally: | |
self.handle_pip_version_check(options) | |
return SUCCESS |
""" | |
shared options and groups | |
The principle here is to define options once, but *not* instantiate them | |
globally. One reason being that options with action='append' can carry state | |
between parses. pip parses general options twice internally, and shouldn't | |
pass on state. To be consistent, all options will follow this design. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import textwrap | |
import warnings | |
from distutils.util import strtobool | |
from functools import partial | |
from optparse import SUPPRESS_HELP, Option, OptionGroup | |
from textwrap import dedent | |
from pip._internal.exceptions import CommandError | |
from pip._internal.locations import USER_CACHE_DIR, get_src_prefix | |
from pip._internal.models.format_control import FormatControl | |
from pip._internal.models.index import PyPI | |
from pip._internal.models.target_python import TargetPython | |
from pip._internal.utils.hashes import STRONG_HASHES | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.ui import BAR_TYPES | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Callable, Dict, Optional, Tuple | |
from optparse import OptionParser, Values | |
from pip._internal.cli.parser import ConfigOptionParser | |
logger = logging.getLogger(__name__) | |
def raise_option_error(parser, option, msg): | |
# type: (OptionParser, Option, str) -> None | |
""" | |
Raise an option parsing error using parser.error(). | |
Args: | |
parser: an OptionParser instance. | |
option: an Option instance. | |
msg: the error text. | |
""" | |
msg = '{} error: {}'.format(option, msg) | |
msg = textwrap.fill(' '.join(msg.split())) | |
parser.error(msg) | |
def make_option_group(group, parser): | |
# type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup | |
""" | |
Return an OptionGroup object | |
group -- assumed to be dict with 'name' and 'options' keys | |
parser -- an optparse Parser | |
""" | |
option_group = OptionGroup(parser, group['name']) | |
for option in group['options']: | |
option_group.add_option(option()) | |
return option_group | |
def check_install_build_global(options, check_options=None): | |
# type: (Values, Optional[Values]) -> None | |
"""Disable wheels if per-setup.py call options are set. | |
:param options: The OptionParser options to update. | |
:param check_options: The options to check, if not supplied defaults to | |
options. | |
""" | |
if check_options is None: | |
check_options = options | |
def getname(n): | |
# type: (str) -> Optional[Any] | |
return getattr(check_options, n, None) | |
names = ["build_options", "global_options", "install_options"] | |
if any(map(getname, names)): | |
control = options.format_control | |
control.disallow_binaries() | |
warnings.warn( | |
'Disabling all use of wheels due to the use of --build-option ' | |
'/ --global-option / --install-option.', stacklevel=2, | |
) | |
def check_dist_restriction(options, check_target=False): | |
# type: (Values, bool) -> None | |
"""Function for determining if custom platform options are allowed. | |
:param options: The OptionParser options. | |
:param check_target: Whether or not to check if --target is being used. | |
""" | |
dist_restriction_set = any([ | |
options.python_version, | |
options.platform, | |
options.abi, | |
options.implementation, | |
]) | |
binary_only = FormatControl(set(), {':all:'}) | |
sdist_dependencies_allowed = ( | |
options.format_control != binary_only and | |
not options.ignore_dependencies | |
) | |
# Installations or downloads using dist restrictions must not combine | |
# source distributions and dist-specific wheels, as they are not | |
# guaranteed to be locally compatible. | |
if dist_restriction_set and sdist_dependencies_allowed: | |
raise CommandError( | |
"When restricting platform and interpreter constraints using " | |
"--python-version, --platform, --abi, or --implementation, " | |
"either --no-deps must be set, or --only-binary=:all: must be " | |
"set and --no-binary must not be set (or must be set to " | |
":none:)." | |
) | |
if check_target: | |
if dist_restriction_set and not options.target_dir: | |
raise CommandError( | |
"Can not use any platform or abi specific options unless " | |
"installing via '--target'" | |
) | |
def _path_option_check(option, opt, value): | |
# type: (Option, str, str) -> str | |
return os.path.expanduser(value) | |
class PipOption(Option): | |
TYPES = Option.TYPES + ("path",) | |
TYPE_CHECKER = Option.TYPE_CHECKER.copy() | |
TYPE_CHECKER["path"] = _path_option_check | |
########### | |
# options # | |
########### | |
help_ = partial( | |
Option, | |
'-h', '--help', | |
dest='help', | |
action='help', | |
help='Show help.', | |
) # type: Callable[..., Option] | |
isolated_mode = partial( | |
Option, | |
"--isolated", | |
dest="isolated_mode", | |
action="store_true", | |
default=False, | |
help=( | |
"Run pip in an isolated mode, ignoring environment variables and user " | |
"configuration." | |
), | |
) # type: Callable[..., Option] | |
require_virtualenv = partial( | |
Option, | |
# Run only if inside a virtualenv, bail if not. | |
'--require-virtualenv', '--require-venv', | |
dest='require_venv', | |
action='store_true', | |
default=False, | |
help=SUPPRESS_HELP | |
) # type: Callable[..., Option] | |
verbose = partial( | |
Option, | |
'-v', '--verbose', | |
dest='verbose', | |
action='count', | |
default=0, | |
help='Give more output. Option is additive, and can be used up to 3 times.' | |
) # type: Callable[..., Option] | |
no_color = partial( | |
Option, | |
'--no-color', | |
dest='no_color', | |
action='store_true', | |
default=False, | |
help="Suppress colored output", | |
) # type: Callable[..., Option] | |
version = partial( | |
Option, | |
'-V', '--version', | |
dest='version', | |
action='store_true', | |
help='Show version and exit.', | |
) # type: Callable[..., Option] | |
quiet = partial( | |
Option, | |
'-q', '--quiet', | |
dest='quiet', | |
action='count', | |
default=0, | |
help=( | |
'Give less output. Option is additive, and can be used up to 3' | |
' times (corresponding to WARNING, ERROR, and CRITICAL logging' | |
' levels).' | |
), | |
) # type: Callable[..., Option] | |
progress_bar = partial( | |
Option, | |
'--progress-bar', | |
dest='progress_bar', | |
type='choice', | |
choices=list(BAR_TYPES.keys()), | |
default='on', | |
help=( | |
'Specify type of progress to be displayed [' + | |
'|'.join(BAR_TYPES.keys()) + '] (default: %default)' | |
), | |
) # type: Callable[..., Option] | |
log = partial( | |
PipOption, | |
"--log", "--log-file", "--local-log", | |
dest="log", | |
metavar="path", | |
type="path", | |
help="Path to a verbose appending log." | |
) # type: Callable[..., Option] | |
no_input = partial( | |
Option, | |
# Don't ask for input | |
'--no-input', | |
dest='no_input', | |
action='store_true', | |
default=False, | |
help=SUPPRESS_HELP | |
) # type: Callable[..., Option] | |
proxy = partial( | |
Option, | |
'--proxy', | |
dest='proxy', | |
type='str', | |
default='', | |
help="Specify a proxy in the form [user:passwd@]proxy.server:port." | |
) # type: Callable[..., Option] | |
retries = partial( | |
Option, | |
'--retries', | |
dest='retries', | |
type='int', | |
default=5, | |
help="Maximum number of retries each connection should attempt " | |
"(default %default times).", | |
) # type: Callable[..., Option] | |
timeout = partial( | |
Option, | |
'--timeout', '--default-timeout', | |
metavar='sec', | |
dest='timeout', | |
type='float', | |
default=15, | |
help='Set the socket timeout (default %default seconds).', | |
) # type: Callable[..., Option] | |
skip_requirements_regex = partial( | |
Option, | |
# A regex to be used to skip requirements | |
'--skip-requirements-regex', | |
dest='skip_requirements_regex', | |
type='str', | |
default='', | |
help=SUPPRESS_HELP, | |
) # type: Callable[..., Option] | |
def exists_action(): | |
# type: () -> Option | |
return Option( | |
# Option when path already exist | |
'--exists-action', | |
dest='exists_action', | |
type='choice', | |
choices=['s', 'i', 'w', 'b', 'a'], | |
default=[], | |
action='append', | |
metavar='action', | |
help="Default action when a path already exists: " | |
"(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.", | |
) | |
cert = partial( | |
PipOption, | |
'--cert', | |
dest='cert', | |
type='path', | |
metavar='path', | |
help="Path to alternate CA bundle.", | |
) # type: Callable[..., Option] | |
client_cert = partial( | |
PipOption, | |
'--client-cert', | |
dest='client_cert', | |
type='path', | |
default=None, | |
metavar='path', | |
help="Path to SSL client certificate, a single file containing the " | |
"private key and the certificate in PEM format.", | |
) # type: Callable[..., Option] | |
index_url = partial( | |
Option, | |
'-i', '--index-url', '--pypi-url', | |
dest='index_url', | |
metavar='URL', | |
default=PyPI.simple_url, | |
help="Base URL of the Python Package Index (default %default). " | |
"This should point to a repository compliant with PEP 503 " | |
"(the simple repository API) or a local directory laid out " | |
"in the same format.", | |
) # type: Callable[..., Option] | |
def extra_index_url(): | |
# type: () -> Option | |
return Option( | |
'--extra-index-url', | |
dest='extra_index_urls', | |
metavar='URL', | |
action='append', | |
default=[], | |
help="Extra URLs of package indexes to use in addition to " | |
"--index-url. Should follow the same rules as " | |
"--index-url.", | |
) | |
no_index = partial( | |
Option, | |
'--no-index', | |
dest='no_index', | |
action='store_true', | |
default=False, | |
help='Ignore package index (only looking at --find-links URLs instead).', | |
) # type: Callable[..., Option] | |
def find_links(): | |
# type: () -> Option | |
return Option( | |
'-f', '--find-links', | |
dest='find_links', | |
action='append', | |
default=[], | |
metavar='url', | |
help="If a url or path to an html file, then parse for links to " | |
"archives. If a local path or file:// url that's a directory, " | |
"then look for archives in the directory listing.", | |
) | |
def trusted_host(): | |
# type: () -> Option | |
return Option( | |
"--trusted-host", | |
dest="trusted_hosts", | |
action="append", | |
metavar="HOSTNAME", | |
default=[], | |
help="Mark this host or host:port pair as trusted, even though it " | |
"does not have valid or any HTTPS.", | |
) | |
def constraints(): | |
# type: () -> Option | |
return Option( | |
'-c', '--constraint', | |
dest='constraints', | |
action='append', | |
default=[], | |
metavar='file', | |
help='Constrain versions using the given constraints file. ' | |
'This option can be used multiple times.' | |
) | |
def requirements(): | |
# type: () -> Option | |
return Option( | |
'-r', '--requirement', | |
dest='requirements', | |
action='append', | |
default=[], | |
metavar='file', | |
help='Install from the given requirements file. ' | |
'This option can be used multiple times.' | |
) | |
def editable(): | |
# type: () -> Option | |
return Option( | |
'-e', '--editable', | |
dest='editables', | |
action='append', | |
default=[], | |
metavar='path/url', | |
help=('Install a project in editable mode (i.e. setuptools ' | |
'"develop mode") from a local project path or a VCS url.'), | |
) | |
def _handle_src(option, opt_str, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
value = os.path.abspath(value) | |
setattr(parser.values, option.dest, value) | |
src = partial( | |
PipOption, | |
'--src', '--source', '--source-dir', '--source-directory', | |
dest='src_dir', | |
type='path', | |
metavar='dir', | |
default=get_src_prefix(), | |
action='callback', | |
callback=_handle_src, | |
help='Directory to check out editable projects into. ' | |
'The default in a virtualenv is "<venv path>/src". ' | |
'The default for global installs is "<current dir>/src".' | |
) # type: Callable[..., Option] | |
def _get_format_control(values, option): | |
# type: (Values, Option) -> Any | |
"""Get a format_control object.""" | |
return getattr(values, option.dest) | |
def _handle_no_binary(option, opt_str, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
existing = _get_format_control(parser.values, option) | |
FormatControl.handle_mutual_excludes( | |
value, existing.no_binary, existing.only_binary, | |
) | |
def _handle_only_binary(option, opt_str, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
existing = _get_format_control(parser.values, option) | |
FormatControl.handle_mutual_excludes( | |
value, existing.only_binary, existing.no_binary, | |
) | |
def no_binary(): | |
# type: () -> Option | |
format_control = FormatControl(set(), set()) | |
return Option( | |
"--no-binary", dest="format_control", action="callback", | |
callback=_handle_no_binary, type="str", | |
default=format_control, | |
help="Do not use binary packages. Can be supplied multiple times, and " | |
"each time adds to the existing value. Accepts either :all: to " | |
"disable all binary packages, :none: to empty the set, or one or " | |
"more package names with commas between them (no colons). Note " | |
"that some packages are tricky to compile and may fail to " | |
"install when this option is used on them.", | |
) | |
def only_binary(): | |
# type: () -> Option | |
format_control = FormatControl(set(), set()) | |
return Option( | |
"--only-binary", dest="format_control", action="callback", | |
callback=_handle_only_binary, type="str", | |
default=format_control, | |
help="Do not use source packages. Can be supplied multiple times, and " | |
"each time adds to the existing value. Accepts either :all: to " | |
"disable all source packages, :none: to empty the set, or one or " | |
"more package names with commas between them. Packages without " | |
"binary distributions will fail to install when this option is " | |
"used on them.", | |
) | |
platform = partial( | |
Option, | |
'--platform', | |
dest='platform', | |
metavar='platform', | |
default=None, | |
help=("Only use wheels compatible with <platform>. " | |
"Defaults to the platform of the running system."), | |
) # type: Callable[..., Option] | |
# This was made a separate function for unit-testing purposes. | |
def _convert_python_version(value): | |
# type: (str) -> Tuple[Tuple[int, ...], Optional[str]] | |
""" | |
Convert a version string like "3", "37", or "3.7.3" into a tuple of ints. | |
:return: A 2-tuple (version_info, error_msg), where `error_msg` is | |
non-None if and only if there was a parsing error. | |
""" | |
if not value: | |
# The empty string is the same as not providing a value. | |
return (None, None) | |
parts = value.split('.') | |
if len(parts) > 3: | |
return ((), 'at most three version parts are allowed') | |
if len(parts) == 1: | |
# Then we are in the case of "3" or "37". | |
value = parts[0] | |
if len(value) > 1: | |
parts = [value[0], value[1:]] | |
try: | |
version_info = tuple(int(part) for part in parts) | |
except ValueError: | |
return ((), 'each version part must be an integer') | |
return (version_info, None) | |
def _handle_python_version(option, opt_str, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
""" | |
Handle a provided --python-version value. | |
""" | |
version_info, error_msg = _convert_python_version(value) | |
if error_msg is not None: | |
msg = ( | |
'invalid --python-version value: {!r}: {}'.format( | |
value, error_msg, | |
) | |
) | |
raise_option_error(parser, option=option, msg=msg) | |
parser.values.python_version = version_info | |
python_version = partial( | |
Option, | |
'--python-version', | |
dest='python_version', | |
metavar='python_version', | |
action='callback', | |
callback=_handle_python_version, type='str', | |
default=None, | |
help=dedent("""\ | |
The Python interpreter version to use for wheel and "Requires-Python" | |
compatibility checks. Defaults to a version derived from the running | |
interpreter. The version can be specified using up to three dot-separated | |
integers (e.g. "3" for 3.0.0, "3.7" for 3.7.0, or "3.7.3"). A major-minor | |
version can also be given as a string without dots (e.g. "37" for 3.7.0). | |
"""), | |
) # type: Callable[..., Option] | |
implementation = partial( | |
Option, | |
'--implementation', | |
dest='implementation', | |
metavar='implementation', | |
default=None, | |
help=("Only use wheels compatible with Python " | |
"implementation <implementation>, e.g. 'pp', 'jy', 'cp', " | |
" or 'ip'. If not specified, then the current " | |
"interpreter implementation is used. Use 'py' to force " | |
"implementation-agnostic wheels."), | |
) # type: Callable[..., Option] | |
abi = partial( | |
Option, | |
'--abi', | |
dest='abi', | |
metavar='abi', | |
default=None, | |
help=("Only use wheels compatible with Python " | |
"abi <abi>, e.g. 'pypy_41'. If not specified, then the " | |
"current interpreter abi tag is used. Generally " | |
"you will need to specify --implementation, " | |
"--platform, and --python-version when using " | |
"this option."), | |
) # type: Callable[..., Option] | |
def add_target_python_options(cmd_opts): | |
# type: (OptionGroup) -> None | |
cmd_opts.add_option(platform()) | |
cmd_opts.add_option(python_version()) | |
cmd_opts.add_option(implementation()) | |
cmd_opts.add_option(abi()) | |
def make_target_python(options): | |
# type: (Values) -> TargetPython | |
target_python = TargetPython( | |
platform=options.platform, | |
py_version_info=options.python_version, | |
abi=options.abi, | |
implementation=options.implementation, | |
) | |
return target_python | |
def prefer_binary(): | |
# type: () -> Option | |
return Option( | |
"--prefer-binary", | |
dest="prefer_binary", | |
action="store_true", | |
default=False, | |
help="Prefer older binary packages over newer source packages." | |
) | |
cache_dir = partial( | |
PipOption, | |
"--cache-dir", | |
dest="cache_dir", | |
default=USER_CACHE_DIR, | |
metavar="dir", | |
type='path', | |
help="Store the cache data in <dir>." | |
) # type: Callable[..., Option] | |
def _handle_no_cache_dir(option, opt, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
""" | |
Process a value provided for the --no-cache-dir option. | |
This is an optparse.Option callback for the --no-cache-dir option. | |
""" | |
# The value argument will be None if --no-cache-dir is passed via the | |
# command-line, since the option doesn't accept arguments. However, | |
# the value can be non-None if the option is triggered e.g. by an | |
# environment variable, like PIP_NO_CACHE_DIR=true. | |
if value is not None: | |
# Then parse the string value to get argument error-checking. | |
try: | |
strtobool(value) | |
except ValueError as exc: | |
raise_option_error(parser, option=option, msg=str(exc)) | |
# Originally, setting PIP_NO_CACHE_DIR to a value that strtobool() | |
# converted to 0 (like "false" or "no") caused cache_dir to be disabled | |
# rather than enabled (logic would say the latter). Thus, we disable | |
# the cache directory not just on values that parse to True, but (for | |
# backwards compatibility reasons) also on values that parse to False. | |
# In other words, always set it to False if the option is provided in | |
# some (valid) form. | |
parser.values.cache_dir = False | |
no_cache = partial( | |
Option, | |
"--no-cache-dir", | |
dest="cache_dir", | |
action="callback", | |
callback=_handle_no_cache_dir, | |
help="Disable the cache.", | |
) # type: Callable[..., Option] | |
no_deps = partial( | |
Option, | |
'--no-deps', '--no-dependencies', | |
dest='ignore_dependencies', | |
action='store_true', | |
default=False, | |
help="Don't install package dependencies.", | |
) # type: Callable[..., Option] | |
def _handle_build_dir(option, opt, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
if value: | |
value = os.path.abspath(value) | |
setattr(parser.values, option.dest, value) | |
build_dir = partial( | |
PipOption, | |
'-b', '--build', '--build-dir', '--build-directory', | |
dest='build_dir', | |
type='path', | |
metavar='dir', | |
action='callback', | |
callback=_handle_build_dir, | |
help='Directory to unpack packages into and build in. Note that ' | |
'an initial build still takes place in a temporary directory. ' | |
'The location of temporary directories can be controlled by setting ' | |
'the TMPDIR environment variable (TEMP on Windows) appropriately. ' | |
'When passed, build directories are not cleaned in case of failures.' | |
) # type: Callable[..., Option] | |
ignore_requires_python = partial( | |
Option, | |
'--ignore-requires-python', | |
dest='ignore_requires_python', | |
action='store_true', | |
help='Ignore the Requires-Python information.' | |
) # type: Callable[..., Option] | |
no_build_isolation = partial( | |
Option, | |
'--no-build-isolation', | |
dest='build_isolation', | |
action='store_false', | |
default=True, | |
help='Disable isolation when building a modern source distribution. ' | |
'Build dependencies specified by PEP 518 must be already installed ' | |
'if this option is used.' | |
) # type: Callable[..., Option] | |
def _handle_no_use_pep517(option, opt, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
""" | |
Process a value provided for the --no-use-pep517 option. | |
This is an optparse.Option callback for the no_use_pep517 option. | |
""" | |
# Since --no-use-pep517 doesn't accept arguments, the value argument | |
# will be None if --no-use-pep517 is passed via the command-line. | |
# However, the value can be non-None if the option is triggered e.g. | |
# by an environment variable, for example "PIP_NO_USE_PEP517=true". | |
if value is not None: | |
msg = """A value was passed for --no-use-pep517, | |
probably using either the PIP_NO_USE_PEP517 environment variable | |
or the "no-use-pep517" config file option. Use an appropriate value | |
of the PIP_USE_PEP517 environment variable or the "use-pep517" | |
config file option instead. | |
""" | |
raise_option_error(parser, option=option, msg=msg) | |
# Otherwise, --no-use-pep517 was passed via the command-line. | |
parser.values.use_pep517 = False | |
use_pep517 = partial( | |
Option, | |
'--use-pep517', | |
dest='use_pep517', | |
action='store_true', | |
default=None, | |
help='Use PEP 517 for building source distributions ' | |
'(use --no-use-pep517 to force legacy behaviour).' | |
) # type: Any | |
no_use_pep517 = partial( | |
Option, | |
'--no-use-pep517', | |
dest='use_pep517', | |
action='callback', | |
callback=_handle_no_use_pep517, | |
default=None, | |
help=SUPPRESS_HELP | |
) # type: Any | |
install_options = partial( | |
Option, | |
'--install-option', | |
dest='install_options', | |
action='append', | |
metavar='options', | |
help="Extra arguments to be supplied to the setup.py install " | |
"command (use like --install-option=\"--install-scripts=/usr/local/" | |
"bin\"). Use multiple --install-option options to pass multiple " | |
"options to setup.py install. If you are using an option with a " | |
"directory path, be sure to use absolute path.", | |
) # type: Callable[..., Option] | |
global_options = partial( | |
Option, | |
'--global-option', | |
dest='global_options', | |
action='append', | |
metavar='options', | |
help="Extra global options to be supplied to the setup.py " | |
"call before the install command.", | |
) # type: Callable[..., Option] | |
no_clean = partial( | |
Option, | |
'--no-clean', | |
action='store_true', | |
default=False, | |
help="Don't clean up build directories." | |
) # type: Callable[..., Option] | |
pre = partial( | |
Option, | |
'--pre', | |
action='store_true', | |
default=False, | |
help="Include pre-release and development versions. By default, " | |
"pip only finds stable versions.", | |
) # type: Callable[..., Option] | |
disable_pip_version_check = partial( | |
Option, | |
"--disable-pip-version-check", | |
dest="disable_pip_version_check", | |
action="store_true", | |
default=True, | |
help="Don't periodically check PyPI to determine whether a new version " | |
"of pip is available for download. Implied with --no-index.", | |
) # type: Callable[..., Option] | |
# Deprecated, Remove later | |
always_unzip = partial( | |
Option, | |
'-Z', '--always-unzip', | |
dest='always_unzip', | |
action='store_true', | |
help=SUPPRESS_HELP, | |
) # type: Callable[..., Option] | |
def _handle_merge_hash(option, opt_str, value, parser): | |
# type: (Option, str, str, OptionParser) -> None | |
"""Given a value spelled "algo:digest", append the digest to a list | |
pointed to in a dict by the algo name.""" | |
if not parser.values.hashes: | |
parser.values.hashes = {} | |
try: | |
algo, digest = value.split(':', 1) | |
except ValueError: | |
parser.error('Arguments to %s must be a hash name ' | |
'followed by a value, like --hash=sha256:abcde...' % | |
opt_str) | |
if algo not in STRONG_HASHES: | |
parser.error('Allowed hash algorithms for %s are %s.' % | |
(opt_str, ', '.join(STRONG_HASHES))) | |
parser.values.hashes.setdefault(algo, []).append(digest) | |
hash = partial( | |
Option, | |
'--hash', | |
# Hash values eventually end up in InstallRequirement.hashes due to | |
# __dict__ copying in process_line(). | |
dest='hashes', | |
action='callback', | |
callback=_handle_merge_hash, | |
type='string', | |
help="Verify that the package's archive matches this " | |
'hash before installing. Example: --hash=sha256:abcdef...', | |
) # type: Callable[..., Option] | |
require_hashes = partial( | |
Option, | |
'--require-hashes', | |
dest='require_hashes', | |
action='store_true', | |
default=False, | |
help='Require a hash to check each requirement against, for ' | |
'repeatable installs. This option is implied when any package in a ' | |
'requirements file has a --hash option.', | |
) # type: Callable[..., Option] | |
list_path = partial( | |
PipOption, | |
'--path', | |
dest='path', | |
type='path', | |
action='append', | |
help='Restrict to the specified installation path for listing ' | |
'packages (can be used multiple times).' | |
) # type: Callable[..., Option] | |
def check_list_path_option(options): | |
# type: (Values) -> None | |
if options.path and (options.user or options.local): | |
raise CommandError( | |
"Cannot combine '--path' with '--user' or '--local'" | |
) | |
no_python_version_warning = partial( | |
Option, | |
'--no-python-version-warning', | |
dest='no_python_version_warning', | |
action='store_true', | |
default=False, | |
help='Silence deprecation warnings for upcoming unsupported Pythons.', | |
) # type: Callable[..., Option] | |
########## | |
# groups # | |
########## | |
general_group = { | |
'name': 'General Options', | |
'options': [ | |
help_, | |
isolated_mode, | |
require_virtualenv, | |
verbose, | |
version, | |
quiet, | |
log, | |
no_input, | |
proxy, | |
retries, | |
timeout, | |
skip_requirements_regex, | |
exists_action, | |
trusted_host, | |
cert, | |
client_cert, | |
cache_dir, | |
no_cache, | |
disable_pip_version_check, | |
no_color, | |
no_python_version_warning, | |
] | |
} # type: Dict[str, Any] | |
index_group = { | |
'name': 'Package Index Options', | |
'options': [ | |
index_url, | |
extra_index_url, | |
no_index, | |
find_links, | |
] | |
} # type: Dict[str, Any] |
from contextlib import contextmanager | |
from pip._vendor.contextlib2 import ExitStack | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Iterator, ContextManager, TypeVar | |
_T = TypeVar('_T', covariant=True) | |
class CommandContextMixIn(object): | |
def __init__(self): | |
# type: () -> None | |
super(CommandContextMixIn, self).__init__() | |
self._in_main_context = False | |
self._main_context = ExitStack() | |
@contextmanager | |
def main_context(self): | |
# type: () -> Iterator[None] | |
assert not self._in_main_context | |
self._in_main_context = True | |
try: | |
with self._main_context: | |
yield | |
finally: | |
self._in_main_context = False | |
def enter_context(self, context_provider): | |
# type: (ContextManager[_T]) -> _T | |
assert self._in_main_context | |
return self._main_context.enter_context(context_provider) |
"""Primary application entrypoint. | |
""" | |
from __future__ import absolute_import | |
import locale | |
import logging | |
import os | |
import sys | |
from pip._internal.cli.autocompletion import autocomplete | |
from pip._internal.cli.main_parser import parse_command | |
from pip._internal.commands import create_command | |
from pip._internal.exceptions import PipError | |
from pip._internal.utils import deprecation | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional | |
logger = logging.getLogger(__name__) | |
# Do not import and use main() directly! Using it directly is actively | |
# discouraged by pip's maintainers. The name, location and behavior of | |
# this function is subject to change, so calling it directly is not | |
# portable across different pip versions. | |
# In addition, running pip in-process is unsupported and unsafe. This is | |
# elaborated in detail at | |
# https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program. | |
# That document also provides suggestions that should work for nearly | |
# all users that are considering importing and using main() directly. | |
# However, we know that certain users will still want to invoke pip | |
# in-process. If you understand and accept the implications of using pip | |
# in an unsupported manner, the best approach is to use runpy to avoid | |
# depending on the exact location of this entry point. | |
# The following example shows how to use runpy to invoke pip in that | |
# case: | |
# | |
# sys.argv = ["pip", your, args, here] | |
# runpy.run_module("pip", run_name="__main__") | |
# | |
# Note that this will exit the process after running, unlike a direct | |
# call to main. As it is not safe to do any processing after calling | |
# main, this should not be an issue in practice. | |
def main(args=None): | |
# type: (Optional[List[str]]) -> int | |
if args is None: | |
args = sys.argv[1:] | |
# Configure our deprecation warnings to be sent through loggers | |
deprecation.install_warning_logger() | |
autocomplete() | |
try: | |
cmd_name, cmd_args = parse_command(args) | |
except PipError as exc: | |
sys.stderr.write("ERROR: %s" % exc) | |
sys.stderr.write(os.linesep) | |
sys.exit(1) | |
# Needed for locale.getpreferredencoding(False) to work | |
# in pip._internal.utils.encoding.auto_decode | |
try: | |
locale.setlocale(locale.LC_ALL, '') | |
except locale.Error as e: | |
# setlocale can apparently crash if locale are uninitialized | |
logger.debug("Ignoring error %s when setting locale", e) | |
command = create_command(cmd_name, isolated=("--isolated" in cmd_args)) | |
return command.main(cmd_args) |
"""A single place for constructing and exposing the main parser | |
""" | |
import os | |
import sys | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.parser import ( | |
ConfigOptionParser, | |
UpdatingDefaultsHelpFormatter, | |
) | |
from pip._internal.commands import commands_dict, get_similar_commands | |
from pip._internal.exceptions import CommandError | |
from pip._internal.utils.misc import get_pip_version, get_prog | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Tuple, List | |
__all__ = ["create_main_parser", "parse_command"] | |
def create_main_parser(): | |
# type: () -> ConfigOptionParser | |
"""Creates and returns the main parser for pip's CLI | |
""" | |
parser_kw = { | |
'usage': '\n%prog <command> [options]', | |
'add_help_option': False, | |
'formatter': UpdatingDefaultsHelpFormatter(), | |
'name': 'global', | |
'prog': get_prog(), | |
} | |
parser = ConfigOptionParser(**parser_kw) | |
parser.disable_interspersed_args() | |
parser.version = get_pip_version() | |
# add the general options | |
gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) | |
parser.add_option_group(gen_opts) | |
# so the help formatter knows | |
parser.main = True # type: ignore | |
# create command listing for description | |
description = [''] + [ | |
'%-27s %s' % (name, command_info.summary) | |
for name, command_info in commands_dict.items() | |
] | |
parser.description = '\n'.join(description) | |
return parser | |
def parse_command(args): | |
# type: (List[str]) -> Tuple[str, List[str]] | |
parser = create_main_parser() | |
# Note: parser calls disable_interspersed_args(), so the result of this | |
# call is to split the initial args into the general options before the | |
# subcommand and everything else. | |
# For example: | |
# args: ['--timeout=5', 'install', '--user', 'INITools'] | |
# general_options: ['--timeout==5'] | |
# args_else: ['install', '--user', 'INITools'] | |
general_options, args_else = parser.parse_args(args) | |
# --version | |
if general_options.version: | |
sys.stdout.write(parser.version) # type: ignore | |
sys.stdout.write(os.linesep) | |
sys.exit() | |
# pip || pip help -> print_help() | |
if not args_else or (args_else[0] == 'help' and len(args_else) == 1): | |
parser.print_help() | |
sys.exit() | |
# the subcommand name | |
cmd_name = args_else[0] | |
if cmd_name not in commands_dict: | |
guess = get_similar_commands(cmd_name) | |
msg = ['unknown command "%s"' % cmd_name] | |
if guess: | |
msg.append('maybe you meant "%s"' % guess) | |
raise CommandError(' - '.join(msg)) | |
# all the args without the subcommand | |
cmd_args = args[:] | |
cmd_args.remove(cmd_name) | |
return cmd_name, cmd_args |
"""Base option parser setup""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import optparse | |
import sys | |
import textwrap | |
from distutils.util import strtobool | |
from pip._vendor.six import string_types | |
from pip._internal.cli.status_codes import UNKNOWN_ERROR | |
from pip._internal.configuration import Configuration, ConfigurationError | |
from pip._internal.utils.compat import get_terminal_size | |
logger = logging.getLogger(__name__) | |
class PrettyHelpFormatter(optparse.IndentedHelpFormatter): | |
"""A prettier/less verbose help formatter for optparse.""" | |
def __init__(self, *args, **kwargs): | |
# help position must be aligned with __init__.parseopts.description | |
kwargs['max_help_position'] = 30 | |
kwargs['indent_increment'] = 1 | |
kwargs['width'] = get_terminal_size()[0] - 2 | |
optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs) | |
def format_option_strings(self, option): | |
return self._format_option_strings(option, ' <%s>', ', ') | |
def _format_option_strings(self, option, mvarfmt=' <%s>', optsep=', '): | |
""" | |
Return a comma-separated list of option strings and metavars. | |
:param option: tuple of (short opt, long opt), e.g: ('-f', '--format') | |
:param mvarfmt: metavar format string - evaluated as mvarfmt % metavar | |
:param optsep: separator | |
""" | |
opts = [] | |
if option._short_opts: | |
opts.append(option._short_opts[0]) | |
if option._long_opts: | |
opts.append(option._long_opts[0]) | |
if len(opts) > 1: | |
opts.insert(1, optsep) | |
if option.takes_value(): | |
metavar = option.metavar or option.dest.lower() | |
opts.append(mvarfmt % metavar.lower()) | |
return ''.join(opts) | |
def format_heading(self, heading): | |
if heading == 'Options': | |
return '' | |
return heading + ':\n' | |
def format_usage(self, usage): | |
""" | |
Ensure there is only one newline between usage and the first heading | |
if there is no description. | |
""" | |
msg = '\nUsage: %s\n' % self.indent_lines(textwrap.dedent(usage), " ") | |
return msg | |
def format_description(self, description): | |
# leave full control over description to us | |
if description: | |
if hasattr(self.parser, 'main'): | |
label = 'Commands' | |
else: | |
label = 'Description' | |
# some doc strings have initial newlines, some don't | |
description = description.lstrip('\n') | |
# some doc strings have final newlines and spaces, some don't | |
description = description.rstrip() | |
# dedent, then reindent | |
description = self.indent_lines(textwrap.dedent(description), " ") | |
description = '%s:\n%s\n' % (label, description) | |
return description | |
else: | |
return '' | |
def format_epilog(self, epilog): | |
# leave full control over epilog to us | |
if epilog: | |
return epilog | |
else: | |
return '' | |
def indent_lines(self, text, indent): | |
new_lines = [indent + line for line in text.split('\n')] | |
return "\n".join(new_lines) | |
class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): | |
"""Custom help formatter for use in ConfigOptionParser. | |
This is updates the defaults before expanding them, allowing | |
them to show up correctly in the help listing. | |
""" | |
def expand_default(self, option): | |
if self.parser is not None: | |
self.parser._update_defaults(self.parser.defaults) | |
return optparse.IndentedHelpFormatter.expand_default(self, option) | |
class CustomOptionParser(optparse.OptionParser): | |
def insert_option_group(self, idx, *args, **kwargs): | |
"""Insert an OptionGroup at a given position.""" | |
group = self.add_option_group(*args, **kwargs) | |
self.option_groups.pop() | |
self.option_groups.insert(idx, group) | |
return group | |
@property | |
def option_list_all(self): | |
"""Get a list of all options, including those in option groups.""" | |
res = self.option_list[:] | |
for i in self.option_groups: | |
res.extend(i.option_list) | |
return res | |
class ConfigOptionParser(CustomOptionParser): | |
"""Custom option parser which updates its defaults by checking the | |
configuration files and environmental variables""" | |
def __init__(self, *args, **kwargs): | |
self.name = kwargs.pop('name') | |
isolated = kwargs.pop("isolated", False) | |
self.config = Configuration(isolated) | |
assert self.name | |
optparse.OptionParser.__init__(self, *args, **kwargs) | |
def check_default(self, option, key, val): | |
try: | |
return option.check_value(key, val) | |
except optparse.OptionValueError as exc: | |
print("An error occurred during configuration: %s" % exc) | |
sys.exit(3) | |
def _get_ordered_configuration_items(self): | |
# Configuration gives keys in an unordered manner. Order them. | |
override_order = ["global", self.name, ":env:"] | |
# Pool the options into different groups | |
section_items = {name: [] for name in override_order} | |
for section_key, val in self.config.items(): | |
# ignore empty values | |
if not val: | |
logger.debug( | |
"Ignoring configuration key '%s' as it's value is empty.", | |
section_key | |
) | |
continue | |
section, key = section_key.split(".", 1) | |
if section in override_order: | |
section_items[section].append((key, val)) | |
# Yield each group in their override order | |
for section in override_order: | |
for key, val in section_items[section]: | |
yield key, val | |
def _update_defaults(self, defaults): | |
"""Updates the given defaults with values from the config files and | |
the environ. Does a little special handling for certain types of | |
options (lists).""" | |
# Accumulate complex default state. | |
self.values = optparse.Values(self.defaults) | |
late_eval = set() | |
# Then set the options with those values | |
for key, val in self._get_ordered_configuration_items(): | |
# '--' because configuration supports only long names | |
option = self.get_option('--' + key) | |
# Ignore options not present in this parser. E.g. non-globals put | |
# in [global] by users that want them to apply to all applicable | |
# commands. | |
if option is None: | |
continue | |
if option.action in ('store_true', 'store_false', 'count'): | |
try: | |
val = strtobool(val) | |
except ValueError: | |
error_msg = invalid_config_error_message( | |
option.action, key, val | |
) | |
self.error(error_msg) | |
elif option.action == 'append': | |
val = val.split() | |
val = [self.check_default(option, key, v) for v in val] | |
elif option.action == 'callback': | |
late_eval.add(option.dest) | |
opt_str = option.get_opt_string() | |
val = option.convert_value(opt_str, val) | |
# From take_action | |
args = option.callback_args or () | |
kwargs = option.callback_kwargs or {} | |
option.callback(option, opt_str, val, self, *args, **kwargs) | |
else: | |
val = self.check_default(option, key, val) | |
defaults[option.dest] = val | |
for key in late_eval: | |
defaults[key] = getattr(self.values, key) | |
self.values = None | |
return defaults | |
def get_default_values(self): | |
"""Overriding to make updating the defaults after instantiation of | |
the option parser possible, _update_defaults() does the dirty work.""" | |
if not self.process_default_values: | |
# Old, pre-Optik 1.5 behaviour. | |
return optparse.Values(self.defaults) | |
# Load the configuration, or error out in case of an error | |
try: | |
self.config.load() | |
except ConfigurationError as err: | |
self.exit(UNKNOWN_ERROR, str(err)) | |
defaults = self._update_defaults(self.defaults.copy()) # ours | |
for option in self._get_all_options(): | |
default = defaults.get(option.dest) | |
if isinstance(default, string_types): | |
opt_str = option.get_opt_string() | |
defaults[option.dest] = option.check_value(opt_str, default) | |
return optparse.Values(defaults) | |
def error(self, msg): | |
self.print_usage(sys.stderr) | |
self.exit(UNKNOWN_ERROR, "%s\n" % msg) | |
def invalid_config_error_message(action, key, val): | |
"""Returns a better error message when invalid configuration option | |
is provided.""" | |
if action in ('store_true', 'store_false'): | |
return ("{0} is not a valid value for {1} option, " | |
"please specify a boolean value like yes/no, " | |
"true/false or 1/0 instead.").format(val, key) | |
return ("{0} is not a valid value for {1} option, " | |
"please specify a numerical value like 1/0 " | |
"instead.").format(val, key) |
"""Contains the Command base classes that depend on PipSession. | |
The classes in this module are in a separate module so the commands not | |
needing download / PackageFinder capability don't unnecessarily import the | |
PackageFinder machinery and all its vendored dependencies, etc. | |
""" | |
import logging | |
import os | |
from functools import partial | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.command_context import CommandContextMixIn | |
from pip._internal.exceptions import CommandError | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.legacy_resolve import Resolver | |
from pip._internal.models.selection_prefs import SelectionPreferences | |
from pip._internal.network.download import Downloader | |
from pip._internal.network.session import PipSession | |
from pip._internal.operations.prepare import RequirementPreparer | |
from pip._internal.req.constructors import ( | |
install_req_from_editable, | |
install_req_from_line, | |
install_req_from_req_string, | |
) | |
from pip._internal.req.req_file import parse_requirements | |
from pip._internal.self_outdated_check import ( | |
make_link_collector, | |
pip_self_version_check, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from optparse import Values | |
from typing import List, Optional, Tuple | |
from pip._internal.cache import WheelCache | |
from pip._internal.models.target_python import TargetPython | |
from pip._internal.req.req_set import RequirementSet | |
from pip._internal.req.req_tracker import RequirementTracker | |
from pip._internal.utils.temp_dir import TempDirectory | |
logger = logging.getLogger(__name__) | |
class SessionCommandMixin(CommandContextMixIn): | |
""" | |
A class mixin for command classes needing _build_session(). | |
""" | |
def __init__(self): | |
# type: () -> None | |
super(SessionCommandMixin, self).__init__() | |
self._session = None # Optional[PipSession] | |
@classmethod | |
def _get_index_urls(cls, options): | |
# type: (Values) -> Optional[List[str]] | |
"""Return a list of index urls from user-provided options.""" | |
index_urls = [] | |
if not getattr(options, "no_index", False): | |
url = getattr(options, "index_url", None) | |
if url: | |
index_urls.append(url) | |
urls = getattr(options, "extra_index_urls", None) | |
if urls: | |
index_urls.extend(urls) | |
# Return None rather than an empty list | |
return index_urls or None | |
def get_default_session(self, options): | |
# type: (Values) -> PipSession | |
"""Get a default-managed session.""" | |
if self._session is None: | |
self._session = self.enter_context(self._build_session(options)) | |
# there's no type annotation on requests.Session, so it's | |
# automatically ContextManager[Any] and self._session becomes Any, | |
# then https://github.com/python/mypy/issues/7696 kicks in | |
assert self._session is not None | |
return self._session | |
def _build_session(self, options, retries=None, timeout=None): | |
# type: (Values, Optional[int], Optional[int]) -> PipSession | |
assert not options.cache_dir or os.path.isabs(options.cache_dir) | |
session = PipSession( | |
cache=( | |
os.path.join(options.cache_dir, "http") | |
if options.cache_dir else None | |
), | |
retries=retries if retries is not None else options.retries, | |
trusted_hosts=options.trusted_hosts, | |
index_urls=self._get_index_urls(options), | |
) | |
# Handle custom ca-bundles from the user | |
if options.cert: | |
session.verify = options.cert | |
# Handle SSL client certificate | |
if options.client_cert: | |
session.cert = options.client_cert | |
# Handle timeouts | |
if options.timeout or timeout: | |
session.timeout = ( | |
timeout if timeout is not None else options.timeout | |
) | |
# Handle configured proxies | |
if options.proxy: | |
session.proxies = { | |
"http": options.proxy, | |
"https": options.proxy, | |
} | |
# Determine if we can prompt the user for authentication or not | |
session.auth.prompting = not options.no_input | |
return session | |
class IndexGroupCommand(Command, SessionCommandMixin): | |
""" | |
Abstract base class for commands with the index_group options. | |
This also corresponds to the commands that permit the pip version check. | |
""" | |
def handle_pip_version_check(self, options): | |
# type: (Values) -> None | |
""" | |
Do the pip version check if not disabled. | |
This overrides the default behavior of not doing the check. | |
""" | |
# Make sure the index_group options are present. | |
assert hasattr(options, 'no_index') | |
if options.disable_pip_version_check or options.no_index: | |
return | |
# Otherwise, check if we're using the latest version of pip available. | |
session = self._build_session( | |
options, | |
retries=0, | |
timeout=min(5, options.timeout) | |
) | |
with session: | |
pip_self_version_check(session, options) | |
class RequirementCommand(IndexGroupCommand): | |
@staticmethod | |
def make_requirement_preparer( | |
temp_build_dir, # type: TempDirectory | |
options, # type: Values | |
req_tracker, # type: RequirementTracker | |
session, # type: PipSession | |
finder, # type: PackageFinder | |
use_user_site, # type: bool | |
download_dir=None, # type: str | |
wheel_download_dir=None, # type: str | |
): | |
# type: (...) -> RequirementPreparer | |
""" | |
Create a RequirementPreparer instance for the given parameters. | |
""" | |
downloader = Downloader(session, progress_bar=options.progress_bar) | |
temp_build_dir_path = temp_build_dir.path | |
assert temp_build_dir_path is not None | |
return RequirementPreparer( | |
build_dir=temp_build_dir_path, | |
src_dir=options.src_dir, | |
download_dir=download_dir, | |
wheel_download_dir=wheel_download_dir, | |
build_isolation=options.build_isolation, | |
req_tracker=req_tracker, | |
downloader=downloader, | |
finder=finder, | |
require_hashes=options.require_hashes, | |
use_user_site=use_user_site, | |
) | |
@staticmethod | |
def make_resolver( | |
preparer, # type: RequirementPreparer | |
finder, # type: PackageFinder | |
options, # type: Values | |
wheel_cache=None, # type: Optional[WheelCache] | |
use_user_site=False, # type: bool | |
ignore_installed=True, # type: bool | |
ignore_requires_python=False, # type: bool | |
force_reinstall=False, # type: bool | |
upgrade_strategy="to-satisfy-only", # type: str | |
use_pep517=None, # type: Optional[bool] | |
py_version_info=None # type: Optional[Tuple[int, ...]] | |
): | |
# type: (...) -> Resolver | |
""" | |
Create a Resolver instance for the given parameters. | |
""" | |
make_install_req = partial( | |
install_req_from_req_string, | |
isolated=options.isolated_mode, | |
wheel_cache=wheel_cache, | |
use_pep517=use_pep517, | |
) | |
return Resolver( | |
preparer=preparer, | |
finder=finder, | |
make_install_req=make_install_req, | |
use_user_site=use_user_site, | |
ignore_dependencies=options.ignore_dependencies, | |
ignore_installed=ignore_installed, | |
ignore_requires_python=ignore_requires_python, | |
force_reinstall=force_reinstall, | |
upgrade_strategy=upgrade_strategy, | |
py_version_info=py_version_info, | |
) | |
def populate_requirement_set( | |
self, | |
requirement_set, # type: RequirementSet | |
args, # type: List[str] | |
options, # type: Values | |
finder, # type: PackageFinder | |
session, # type: PipSession | |
wheel_cache, # type: Optional[WheelCache] | |
): | |
# type: (...) -> None | |
""" | |
Marshal cmd line args into a requirement set. | |
""" | |
for filename in options.constraints: | |
for req_to_add in parse_requirements( | |
filename, | |
constraint=True, finder=finder, options=options, | |
session=session, wheel_cache=wheel_cache): | |
req_to_add.is_direct = True | |
requirement_set.add_requirement(req_to_add) | |
for req in args: | |
req_to_add = install_req_from_line( | |
req, None, isolated=options.isolated_mode, | |
use_pep517=options.use_pep517, | |
wheel_cache=wheel_cache | |
) | |
req_to_add.is_direct = True | |
requirement_set.add_requirement(req_to_add) | |
for req in options.editables: | |
req_to_add = install_req_from_editable( | |
req, | |
isolated=options.isolated_mode, | |
use_pep517=options.use_pep517, | |
wheel_cache=wheel_cache | |
) | |
req_to_add.is_direct = True | |
requirement_set.add_requirement(req_to_add) | |
# NOTE: options.require_hashes may be set if --require-hashes is True | |
for filename in options.requirements: | |
for req_to_add in parse_requirements( | |
filename, | |
finder=finder, options=options, session=session, | |
wheel_cache=wheel_cache, | |
use_pep517=options.use_pep517): | |
req_to_add.is_direct = True | |
requirement_set.add_requirement(req_to_add) | |
# If any requirement has hash options, enable hash checking. | |
requirements = ( | |
requirement_set.unnamed_requirements + | |
list(requirement_set.requirements.values()) | |
) | |
if any(req.has_hash_options for req in requirements): | |
options.require_hashes = True | |
if not (args or options.editables or options.requirements): | |
opts = {'name': self.name} | |
if options.find_links: | |
raise CommandError( | |
'You must give at least one requirement to %(name)s ' | |
'(maybe you meant "pip %(name)s %(links)s"?)' % | |
dict(opts, links=' '.join(options.find_links))) | |
else: | |
raise CommandError( | |
'You must give at least one requirement to %(name)s ' | |
'(see "pip help %(name)s")' % opts) | |
@staticmethod | |
def trace_basic_info(finder): | |
# type: (PackageFinder) -> None | |
""" | |
Trace basic information about the provided objects. | |
""" | |
# Display where finder is looking for packages | |
search_scope = finder.search_scope | |
locations = search_scope.get_formatted_locations() | |
if locations: | |
logger.info(locations) | |
def _build_package_finder( | |
self, | |
options, # type: Values | |
session, # type: PipSession | |
target_python=None, # type: Optional[TargetPython] | |
ignore_requires_python=None, # type: Optional[bool] | |
): | |
# type: (...) -> PackageFinder | |
""" | |
Create a package finder appropriate to this requirement command. | |
:param ignore_requires_python: Whether to ignore incompatible | |
"Requires-Python" values in links. Defaults to False. | |
""" | |
link_collector = make_link_collector(session, options=options) | |
selection_prefs = SelectionPreferences( | |
allow_yanked=True, | |
format_control=options.format_control, | |
allow_all_prereleases=options.pre, | |
prefer_binary=options.prefer_binary, | |
ignore_requires_python=ignore_requires_python, | |
) | |
return PackageFinder.create( | |
link_collector=link_collector, | |
selection_prefs=selection_prefs, | |
target_python=target_python, | |
) |
from __future__ import absolute_import | |
SUCCESS = 0 | |
ERROR = 1 | |
UNKNOWN_ERROR = 2 | |
VIRTUALENV_NOT_FOUND = 3 | |
PREVIOUS_BUILD_DIR_ERROR = 4 | |
NO_MATCHES_FOUND = 23 |
""" | |
Package containing all pip commands | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import importlib | |
from collections import OrderedDict, namedtuple | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any | |
from pip._internal.cli.base_command import Command | |
CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary') | |
# The ordering matters for help display. | |
# Also, even though the module path starts with the same | |
# "pip._internal.commands" prefix in each case, we include the full path | |
# because it makes testing easier (specifically when modifying commands_dict | |
# in test setup / teardown by adding info for a FakeCommand class defined | |
# in a test-related module). | |
# Finally, we need to pass an iterable of pairs here rather than a dict | |
# so that the ordering won't be lost when using Python 2.7. | |
commands_dict = OrderedDict([ | |
('install', CommandInfo( | |
'pip._internal.commands.install', 'InstallCommand', | |
'Install packages.', | |
)), | |
('download', CommandInfo( | |
'pip._internal.commands.download', 'DownloadCommand', | |
'Download packages.', | |
)), | |
('uninstall', CommandInfo( | |
'pip._internal.commands.uninstall', 'UninstallCommand', | |
'Uninstall packages.', | |
)), | |
('freeze', CommandInfo( | |
'pip._internal.commands.freeze', 'FreezeCommand', | |
'Output installed packages in requirements format.', | |
)), | |
('list', CommandInfo( | |
'pip._internal.commands.list', 'ListCommand', | |
'List installed packages.', | |
)), | |
('show', CommandInfo( | |
'pip._internal.commands.show', 'ShowCommand', | |
'Show information about installed packages.', | |
)), | |
('check', CommandInfo( | |
'pip._internal.commands.check', 'CheckCommand', | |
'Verify installed packages have compatible dependencies.', | |
)), | |
('config', CommandInfo( | |
'pip._internal.commands.configuration', 'ConfigurationCommand', | |
'Manage local and global configuration.', | |
)), | |
('search', CommandInfo( | |
'pip._internal.commands.search', 'SearchCommand', | |
'Search PyPI for packages.', | |
)), | |
('wheel', CommandInfo( | |
'pip._internal.commands.wheel', 'WheelCommand', | |
'Build wheels from your requirements.', | |
)), | |
('hash', CommandInfo( | |
'pip._internal.commands.hash', 'HashCommand', | |
'Compute hashes of package archives.', | |
)), | |
('completion', CommandInfo( | |
'pip._internal.commands.completion', 'CompletionCommand', | |
'A helper command used for command completion.', | |
)), | |
('debug', CommandInfo( | |
'pip._internal.commands.debug', 'DebugCommand', | |
'Show information useful for debugging.', | |
)), | |
('help', CommandInfo( | |
'pip._internal.commands.help', 'HelpCommand', | |
'Show help for commands.', | |
)), | |
]) # type: OrderedDict[str, CommandInfo] | |
def create_command(name, **kwargs): | |
# type: (str, **Any) -> Command | |
""" | |
Create an instance of the Command class with the given name. | |
""" | |
module_path, class_name, summary = commands_dict[name] | |
module = importlib.import_module(module_path) | |
command_class = getattr(module, class_name) | |
command = command_class(name=name, summary=summary, **kwargs) | |
return command | |
def get_similar_commands(name): | |
"""Command name auto-correct.""" | |
from difflib import get_close_matches | |
name = name.lower() | |
close_commands = get_close_matches(name, commands_dict.keys()) | |
if close_commands: | |
return close_commands[0] | |
else: | |
return False |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import logging | |
from pip._internal.cli.base_command import Command | |
from pip._internal.operations.check import ( | |
check_package_set, | |
create_package_set_from_installed, | |
) | |
from pip._internal.utils.misc import write_output | |
logger = logging.getLogger(__name__) | |
class CheckCommand(Command): | |
"""Verify installed packages have compatible dependencies.""" | |
usage = """ | |
%prog [options]""" | |
def run(self, options, args): | |
package_set, parsing_probs = create_package_set_from_installed() | |
missing, conflicting = check_package_set(package_set) | |
for project_name in missing: | |
version = package_set[project_name].version | |
for dependency in missing[project_name]: | |
write_output( | |
"%s %s requires %s, which is not installed.", | |
project_name, version, dependency[0], | |
) | |
for project_name in conflicting: | |
version = package_set[project_name].version | |
for dep_name, dep_version, req in conflicting[project_name]: | |
write_output( | |
"%s %s has requirement %s, but you have %s %s.", | |
project_name, version, req, dep_name, dep_version, | |
) | |
if missing or conflicting or parsing_probs: | |
return 1 | |
else: | |
write_output("No broken requirements found.") |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import sys | |
import textwrap | |
from pip._internal.cli.base_command import Command | |
from pip._internal.utils.misc import get_prog | |
BASE_COMPLETION = """ | |
# pip %(shell)s completion start%(script)s# pip %(shell)s completion end | |
""" | |
COMPLETION_SCRIPTS = { | |
'bash': """ | |
_pip_completion() | |
{ | |
COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ | |
COMP_CWORD=$COMP_CWORD \\ | |
PIP_AUTO_COMPLETE=1 $1 2>/dev/null ) ) | |
} | |
complete -o default -F _pip_completion %(prog)s | |
""", | |
'zsh': """ | |
function _pip_completion { | |
local words cword | |
read -Ac words | |
read -cn cword | |
reply=( $( COMP_WORDS="$words[*]" \\ | |
COMP_CWORD=$(( cword-1 )) \\ | |
PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) | |
} | |
compctl -K _pip_completion %(prog)s | |
""", | |
'fish': """ | |
function __fish_complete_pip | |
set -lx COMP_WORDS (commandline -o) "" | |
set -lx COMP_CWORD ( \\ | |
math (contains -i -- (commandline -t) $COMP_WORDS)-1 \\ | |
) | |
set -lx PIP_AUTO_COMPLETE 1 | |
string split \\ -- (eval $COMP_WORDS[1]) | |
end | |
complete -fa "(__fish_complete_pip)" -c %(prog)s | |
""", | |
} | |
class CompletionCommand(Command): | |
"""A helper command to be used for command completion.""" | |
ignore_require_venv = True | |
def __init__(self, *args, **kw): | |
super(CompletionCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmd_opts.add_option( | |
'--bash', '-b', | |
action='store_const', | |
const='bash', | |
dest='shell', | |
help='Emit completion code for bash') | |
cmd_opts.add_option( | |
'--zsh', '-z', | |
action='store_const', | |
const='zsh', | |
dest='shell', | |
help='Emit completion code for zsh') | |
cmd_opts.add_option( | |
'--fish', '-f', | |
action='store_const', | |
const='fish', | |
dest='shell', | |
help='Emit completion code for fish') | |
self.parser.insert_option_group(0, cmd_opts) | |
def run(self, options, args): | |
"""Prints the completion code of the given shell""" | |
shells = COMPLETION_SCRIPTS.keys() | |
shell_options = ['--' + shell for shell in sorted(shells)] | |
if options.shell in shells: | |
script = textwrap.dedent( | |
COMPLETION_SCRIPTS.get(options.shell, '') % { | |
'prog': get_prog(), | |
} | |
) | |
print(BASE_COMPLETION % {'script': script, 'shell': options.shell}) | |
else: | |
sys.stderr.write( | |
'ERROR: You must pass %s\n' % ' or '.join(shell_options) | |
) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import logging | |
import os | |
import subprocess | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.status_codes import ERROR, SUCCESS | |
from pip._internal.configuration import ( | |
Configuration, | |
get_configuration_files, | |
kinds, | |
) | |
from pip._internal.exceptions import PipError | |
from pip._internal.utils.misc import get_prog, write_output | |
logger = logging.getLogger(__name__) | |
class ConfigurationCommand(Command): | |
"""Manage local and global configuration. | |
Subcommands: | |
list: List the active configuration (or from the file specified) | |
edit: Edit the configuration file in an editor | |
get: Get the value associated with name | |
set: Set the name=value | |
unset: Unset the value associated with name | |
If none of --user, --global and --site are passed, a virtual | |
environment configuration file is used if one is active and the file | |
exists. Otherwise, all modifications happen on the to the user file by | |
default. | |
""" | |
ignore_require_venv = True | |
usage = """ | |
%prog [<file-option>] list | |
%prog [<file-option>] [--editor <editor-path>] edit | |
%prog [<file-option>] get name | |
%prog [<file-option>] set name value | |
%prog [<file-option>] unset name | |
""" | |
def __init__(self, *args, **kwargs): | |
super(ConfigurationCommand, self).__init__(*args, **kwargs) | |
self.configuration = None | |
self.cmd_opts.add_option( | |
'--editor', | |
dest='editor', | |
action='store', | |
default=None, | |
help=( | |
'Editor to use to edit the file. Uses VISUAL or EDITOR ' | |
'environment variables if not provided.' | |
) | |
) | |
self.cmd_opts.add_option( | |
'--global', | |
dest='global_file', | |
action='store_true', | |
default=False, | |
help='Use the system-wide configuration file only' | |
) | |
self.cmd_opts.add_option( | |
'--user', | |
dest='user_file', | |
action='store_true', | |
default=False, | |
help='Use the user configuration file only' | |
) | |
self.cmd_opts.add_option( | |
'--site', | |
dest='site_file', | |
action='store_true', | |
default=False, | |
help='Use the current environment configuration file only' | |
) | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
handlers = { | |
"list": self.list_values, | |
"edit": self.open_in_editor, | |
"get": self.get_name, | |
"set": self.set_name_value, | |
"unset": self.unset_name | |
} | |
# Determine action | |
if not args or args[0] not in handlers: | |
logger.error("Need an action ({}) to perform.".format( | |
", ".join(sorted(handlers))) | |
) | |
return ERROR | |
action = args[0] | |
# Determine which configuration files are to be loaded | |
# Depends on whether the command is modifying. | |
try: | |
load_only = self._determine_file( | |
options, need_value=(action in ["get", "set", "unset", "edit"]) | |
) | |
except PipError as e: | |
logger.error(e.args[0]) | |
return ERROR | |
# Load a new configuration | |
self.configuration = Configuration( | |
isolated=options.isolated_mode, load_only=load_only | |
) | |
self.configuration.load() | |
# Error handling happens here, not in the action-handlers. | |
try: | |
handlers[action](options, args[1:]) | |
except PipError as e: | |
logger.error(e.args[0]) | |
return ERROR | |
return SUCCESS | |
def _determine_file(self, options, need_value): | |
file_options = [key for key, value in ( | |
(kinds.USER, options.user_file), | |
(kinds.GLOBAL, options.global_file), | |
(kinds.SITE, options.site_file), | |
) if value] | |
if not file_options: | |
if not need_value: | |
return None | |
# Default to user, unless there's a site file. | |
elif any( | |
os.path.exists(site_config_file) | |
for site_config_file in get_configuration_files()[kinds.SITE] | |
): | |
return kinds.SITE | |
else: | |
return kinds.USER | |
elif len(file_options) == 1: | |
return file_options[0] | |
raise PipError( | |
"Need exactly one file to operate upon " | |
"(--user, --site, --global) to perform." | |
) | |
def list_values(self, options, args): | |
self._get_n_args(args, "list", n=0) | |
for key, value in sorted(self.configuration.items()): | |
write_output("%s=%r", key, value) | |
def get_name(self, options, args): | |
key = self._get_n_args(args, "get [name]", n=1) | |
value = self.configuration.get_value(key) | |
write_output("%s", value) | |
def set_name_value(self, options, args): | |
key, value = self._get_n_args(args, "set [name] [value]", n=2) | |
self.configuration.set_value(key, value) | |
self._save_configuration() | |
def unset_name(self, options, args): | |
key = self._get_n_args(args, "unset [name]", n=1) | |
self.configuration.unset_value(key) | |
self._save_configuration() | |
def open_in_editor(self, options, args): | |
editor = self._determine_editor(options) | |
fname = self.configuration.get_file_to_edit() | |
if fname is None: | |
raise PipError("Could not determine appropriate file.") | |
try: | |
subprocess.check_call([editor, fname]) | |
except subprocess.CalledProcessError as e: | |
raise PipError( | |
"Editor Subprocess exited with exit code {}" | |
.format(e.returncode) | |
) | |
def _get_n_args(self, args, example, n): | |
"""Helper to make sure the command got the right number of arguments | |
""" | |
if len(args) != n: | |
msg = ( | |
'Got unexpected number of arguments, expected {}. ' | |
'(example: "{} config {}")' | |
).format(n, get_prog(), example) | |
raise PipError(msg) | |
if n == 1: | |
return args[0] | |
else: | |
return args | |
def _save_configuration(self): | |
# We successfully ran a modifying command. Need to save the | |
# configuration. | |
try: | |
self.configuration.save() | |
except Exception: | |
logger.error( | |
"Unable to save configuration. Please report this as a bug.", | |
exc_info=1 | |
) | |
raise PipError("Internal Error.") | |
def _determine_editor(self, options): | |
if options.editor is not None: | |
return options.editor | |
elif "VISUAL" in os.environ: | |
return os.environ["VISUAL"] | |
elif "EDITOR" in os.environ: | |
return os.environ["EDITOR"] | |
else: | |
raise PipError("Could not determine editor to use.") |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import locale | |
import logging | |
import os | |
import sys | |
from pip._vendor.certifi import where | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.cmdoptions import make_target_python | |
from pip._internal.cli.status_codes import SUCCESS | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import get_pip_version | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, List, Optional | |
from optparse import Values | |
logger = logging.getLogger(__name__) | |
def show_value(name, value): | |
# type: (str, Optional[str]) -> None | |
logger.info('{}: {}'.format(name, value)) | |
def show_sys_implementation(): | |
# type: () -> None | |
logger.info('sys.implementation:') | |
if hasattr(sys, 'implementation'): | |
implementation = sys.implementation # type: ignore | |
implementation_name = implementation.name | |
else: | |
implementation_name = '' | |
with indent_log(): | |
show_value('name', implementation_name) | |
def show_tags(options): | |
# type: (Values) -> None | |
tag_limit = 10 | |
target_python = make_target_python(options) | |
tags = target_python.get_tags() | |
# Display the target options that were explicitly provided. | |
formatted_target = target_python.format_given() | |
suffix = '' | |
if formatted_target: | |
suffix = ' (target: {})'.format(formatted_target) | |
msg = 'Compatible tags: {}{}'.format(len(tags), suffix) | |
logger.info(msg) | |
if options.verbose < 1 and len(tags) > tag_limit: | |
tags_limited = True | |
tags = tags[:tag_limit] | |
else: | |
tags_limited = False | |
with indent_log(): | |
for tag in tags: | |
logger.info(str(tag)) | |
if tags_limited: | |
msg = ( | |
'...\n' | |
'[First {tag_limit} tags shown. Pass --verbose to show all.]' | |
).format(tag_limit=tag_limit) | |
logger.info(msg) | |
def ca_bundle_info(config): | |
levels = set() | |
for key, value in config.items(): | |
levels.add(key.split('.')[0]) | |
if not levels: | |
return "Not specified" | |
levels_that_override_global = ['install', 'wheel', 'download'] | |
global_overriding_level = [ | |
level for level in levels if level in levels_that_override_global | |
] | |
if not global_overriding_level: | |
return 'global' | |
levels.remove('global') | |
return ", ".join(levels) | |
class DebugCommand(Command): | |
""" | |
Display debug information. | |
""" | |
usage = """ | |
%prog <options>""" | |
ignore_require_venv = True | |
def __init__(self, *args, **kw): | |
super(DebugCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmdoptions.add_target_python_options(cmd_opts) | |
self.parser.insert_option_group(0, cmd_opts) | |
self.parser.config.load() | |
def run(self, options, args): | |
# type: (Values, List[Any]) -> int | |
logger.warning( | |
"This command is only meant for debugging. " | |
"Do not use this with automation for parsing and getting these " | |
"details, since the output and options of this command may " | |
"change without notice." | |
) | |
show_value('pip version', get_pip_version()) | |
show_value('sys.version', sys.version) | |
show_value('sys.executable', sys.executable) | |
show_value('sys.getdefaultencoding', sys.getdefaultencoding()) | |
show_value('sys.getfilesystemencoding', sys.getfilesystemencoding()) | |
show_value( | |
'locale.getpreferredencoding', locale.getpreferredencoding(), | |
) | |
show_value('sys.platform', sys.platform) | |
show_sys_implementation() | |
show_value("'cert' config value", ca_bundle_info(self.parser.config)) | |
show_value("REQUESTS_CA_BUNDLE", os.environ.get('REQUESTS_CA_BUNDLE')) | |
show_value("CURL_CA_BUNDLE", os.environ.get('CURL_CA_BUNDLE')) | |
show_value("pip._vendor.certifi.where()", where()) | |
show_tags(options) | |
return SUCCESS |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.cmdoptions import make_target_python | |
from pip._internal.cli.req_command import RequirementCommand | |
from pip._internal.req import RequirementSet | |
from pip._internal.req.req_tracker import get_requirement_tracker | |
from pip._internal.utils.misc import ensure_dir, normalize_path, write_output | |
from pip._internal.utils.temp_dir import TempDirectory | |
logger = logging.getLogger(__name__) | |
class DownloadCommand(RequirementCommand): | |
""" | |
Download packages from: | |
- PyPI (and other indexes) using requirement specifiers. | |
- VCS project urls. | |
- Local project directories. | |
- Local or remote source archives. | |
pip also supports downloading from "requirements files", which provide | |
an easy way to specify a whole environment to be downloaded. | |
""" | |
usage = """ | |
%prog [options] <requirement specifier> [package-index-options] ... | |
%prog [options] -r <requirements file> [package-index-options] ... | |
%prog [options] <vcs project url> ... | |
%prog [options] <local project path> ... | |
%prog [options] <archive url/path> ...""" | |
def __init__(self, *args, **kw): | |
super(DownloadCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmd_opts.add_option(cmdoptions.constraints()) | |
cmd_opts.add_option(cmdoptions.requirements()) | |
cmd_opts.add_option(cmdoptions.build_dir()) | |
cmd_opts.add_option(cmdoptions.no_deps()) | |
cmd_opts.add_option(cmdoptions.global_options()) | |
cmd_opts.add_option(cmdoptions.no_binary()) | |
cmd_opts.add_option(cmdoptions.only_binary()) | |
cmd_opts.add_option(cmdoptions.prefer_binary()) | |
cmd_opts.add_option(cmdoptions.src()) | |
cmd_opts.add_option(cmdoptions.pre()) | |
cmd_opts.add_option(cmdoptions.no_clean()) | |
cmd_opts.add_option(cmdoptions.require_hashes()) | |
cmd_opts.add_option(cmdoptions.progress_bar()) | |
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |
cmd_opts.add_option(cmdoptions.use_pep517()) | |
cmd_opts.add_option(cmdoptions.no_use_pep517()) | |
cmd_opts.add_option( | |
'-d', '--dest', '--destination-dir', '--destination-directory', | |
dest='download_dir', | |
metavar='dir', | |
default=os.curdir, | |
help=("Download packages into <dir>."), | |
) | |
cmdoptions.add_target_python_options(cmd_opts) | |
index_opts = cmdoptions.make_option_group( | |
cmdoptions.index_group, | |
self.parser, | |
) | |
self.parser.insert_option_group(0, index_opts) | |
self.parser.insert_option_group(0, cmd_opts) | |
def run(self, options, args): | |
options.ignore_installed = True | |
# editable doesn't really make sense for `pip download`, but the bowels | |
# of the RequirementSet code require that property. | |
options.editables = [] | |
cmdoptions.check_dist_restriction(options) | |
options.download_dir = normalize_path(options.download_dir) | |
ensure_dir(options.download_dir) | |
session = self.get_default_session(options) | |
target_python = make_target_python(options) | |
finder = self._build_package_finder( | |
options=options, | |
session=session, | |
target_python=target_python, | |
) | |
build_delete = (not (options.no_clean or options.build_dir)) | |
with get_requirement_tracker() as req_tracker, TempDirectory( | |
options.build_dir, delete=build_delete, kind="download" | |
) as directory: | |
requirement_set = RequirementSet() | |
self.populate_requirement_set( | |
requirement_set, | |
args, | |
options, | |
finder, | |
session, | |
None | |
) | |
preparer = self.make_requirement_preparer( | |
temp_build_dir=directory, | |
options=options, | |
req_tracker=req_tracker, | |
session=session, | |
finder=finder, | |
download_dir=options.download_dir, | |
use_user_site=False, | |
) | |
resolver = self.make_resolver( | |
preparer=preparer, | |
finder=finder, | |
options=options, | |
py_version_info=options.python_version, | |
) | |
self.trace_basic_info(finder) | |
resolver.resolve(requirement_set) | |
downloaded = ' '.join([ | |
req.name for req in requirement_set.successfully_downloaded | |
]) | |
if downloaded: | |
write_output('Successfully downloaded %s', downloaded) | |
# Clean up | |
if not options.no_clean: | |
requirement_set.cleanup_files() | |
return requirement_set |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import sys | |
from pip._internal.cache import WheelCache | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.base_command import Command | |
from pip._internal.models.format_control import FormatControl | |
from pip._internal.operations.freeze import freeze | |
from pip._internal.utils.compat import stdlib_pkgs | |
DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel', 'pkg-resources'} | |
class FreezeCommand(Command): | |
""" | |
Output installed packages in requirements format. | |
packages are listed in a case-insensitive sorted order. | |
""" | |
usage = """ | |
%prog [options]""" | |
log_streams = ("ext://sys.stderr", "ext://sys.stderr") | |
def __init__(self, *args, **kw): | |
super(FreezeCommand, self).__init__(*args, **kw) | |
self.cmd_opts.add_option( | |
'-r', '--requirement', | |
dest='requirements', | |
action='append', | |
default=[], | |
metavar='file', | |
help="Use the order in the given requirements file and its " | |
"comments when generating output. This option can be " | |
"used multiple times.") | |
self.cmd_opts.add_option( | |
'-f', '--find-links', | |
dest='find_links', | |
action='append', | |
default=[], | |
metavar='URL', | |
help='URL for finding packages, which will be added to the ' | |
'output.') | |
self.cmd_opts.add_option( | |
'-l', '--local', | |
dest='local', | |
action='store_true', | |
default=False, | |
help='If in a virtualenv that has global access, do not output ' | |
'globally-installed packages.') | |
self.cmd_opts.add_option( | |
'--user', | |
dest='user', | |
action='store_true', | |
default=False, | |
help='Only output packages installed in user-site.') | |
self.cmd_opts.add_option(cmdoptions.list_path()) | |
self.cmd_opts.add_option( | |
'--all', | |
dest='freeze_all', | |
action='store_true', | |
help='Do not skip these packages in the output:' | |
' %s' % ', '.join(DEV_PKGS)) | |
self.cmd_opts.add_option( | |
'--exclude-editable', | |
dest='exclude_editable', | |
action='store_true', | |
help='Exclude editable package from output.') | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
format_control = FormatControl(set(), set()) | |
wheel_cache = WheelCache(options.cache_dir, format_control) | |
skip = set(stdlib_pkgs) | |
if not options.freeze_all: | |
skip.update(DEV_PKGS) | |
cmdoptions.check_list_path_option(options) | |
freeze_kwargs = dict( | |
requirement=options.requirements, | |
find_links=options.find_links, | |
local_only=options.local, | |
user_only=options.user, | |
paths=options.path, | |
skip_regex=options.skip_requirements_regex, | |
isolated=options.isolated_mode, | |
wheel_cache=wheel_cache, | |
skip=skip, | |
exclude_editable=options.exclude_editable, | |
) | |
try: | |
for line in freeze(**freeze_kwargs): | |
sys.stdout.write(line + '\n') | |
finally: | |
wheel_cache.cleanup() |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import hashlib | |
import logging | |
import sys | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.status_codes import ERROR | |
from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES | |
from pip._internal.utils.misc import read_chunks, write_output | |
logger = logging.getLogger(__name__) | |
class HashCommand(Command): | |
""" | |
Compute a hash of a local package archive. | |
These can be used with --hash in a requirements file to do repeatable | |
installs. | |
""" | |
usage = '%prog [options] <file> ...' | |
ignore_require_venv = True | |
def __init__(self, *args, **kw): | |
super(HashCommand, self).__init__(*args, **kw) | |
self.cmd_opts.add_option( | |
'-a', '--algorithm', | |
dest='algorithm', | |
choices=STRONG_HASHES, | |
action='store', | |
default=FAVORITE_HASH, | |
help='The hash algorithm to use: one of %s' % | |
', '.join(STRONG_HASHES)) | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
if not args: | |
self.parser.print_usage(sys.stderr) | |
return ERROR | |
algorithm = options.algorithm | |
for path in args: | |
write_output('%s:\n--hash=%s:%s', | |
path, algorithm, _hash_of_file(path, algorithm)) | |
def _hash_of_file(path, algorithm): | |
"""Return the hash digest of a file.""" | |
with open(path, 'rb') as archive: | |
hash = hashlib.new(algorithm) | |
for chunk in read_chunks(archive): | |
hash.update(chunk) | |
return hash.hexdigest() |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.status_codes import SUCCESS | |
from pip._internal.exceptions import CommandError | |
class HelpCommand(Command): | |
"""Show help for commands""" | |
usage = """ | |
%prog <command>""" | |
ignore_require_venv = True | |
def run(self, options, args): | |
from pip._internal.commands import ( | |
commands_dict, create_command, get_similar_commands, | |
) | |
try: | |
# 'pip help' with no args is handled by pip.__init__.parseopt() | |
cmd_name = args[0] # the command we need help for | |
except IndexError: | |
return SUCCESS | |
if cmd_name not in commands_dict: | |
guess = get_similar_commands(cmd_name) | |
msg = ['unknown command "%s"' % cmd_name] | |
if guess: | |
msg.append('maybe you meant "%s"' % guess) | |
raise CommandError(' - '.join(msg)) | |
command = create_command(cmd_name) | |
command.parser.print_help() | |
return SUCCESS |
# The following comment should be removed at some point in the future. | |
# It's included for now because without it InstallCommand.run() has a | |
# couple errors where we have to know req.name is str rather than | |
# Optional[str] for the InstallRequirement req. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import errno | |
import logging | |
import operator | |
import os | |
import shutil | |
import site | |
from optparse import SUPPRESS_HELP | |
from pip._vendor import pkg_resources | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal.cache import WheelCache | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.cmdoptions import make_target_python | |
from pip._internal.cli.req_command import RequirementCommand | |
from pip._internal.cli.status_codes import ERROR, SUCCESS | |
from pip._internal.exceptions import ( | |
CommandError, | |
InstallationError, | |
PreviousBuildDirError, | |
) | |
from pip._internal.locations import distutils_scheme | |
from pip._internal.operations.check import check_install_conflicts | |
from pip._internal.req import RequirementSet, install_given_reqs | |
from pip._internal.req.req_tracker import get_requirement_tracker | |
from pip._internal.utils.deprecation import deprecated | |
from pip._internal.utils.distutils_args import parse_distutils_args | |
from pip._internal.utils.filesystem import test_writable_dir | |
from pip._internal.utils.misc import ( | |
ensure_dir, | |
get_installed_version, | |
protect_pip_from_modification_on_windows, | |
write_output, | |
) | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.virtualenv import virtualenv_no_global | |
from pip._internal.wheel_builder import build, should_build_for_install_command | |
if MYPY_CHECK_RUNNING: | |
from optparse import Values | |
from typing import Any, Iterable, List, Optional | |
from pip._internal.models.format_control import FormatControl | |
from pip._internal.req.req_install import InstallRequirement | |
from pip._internal.wheel_builder import BinaryAllowedPredicate | |
from pip._internal.locations import running_under_virtualenv | |
logger = logging.getLogger(__name__) | |
def get_check_binary_allowed(format_control): | |
# type: (FormatControl) -> BinaryAllowedPredicate | |
def check_binary_allowed(req): | |
# type: (InstallRequirement) -> bool | |
if req.use_pep517: | |
return True | |
canonical_name = canonicalize_name(req.name) | |
allowed_formats = format_control.get_allowed_formats(canonical_name) | |
return "binary" in allowed_formats | |
return check_binary_allowed | |
class InstallCommand(RequirementCommand): | |
""" | |
Install packages from: | |
- PyPI (and other indexes) using requirement specifiers. | |
- VCS project urls. | |
- Local project directories. | |
- Local or remote source archives. | |
pip also supports installing from "requirements files", which provide | |
an easy way to specify a whole environment to be installed. | |
""" | |
usage = """ | |
%prog [options] <requirement specifier> [package-index-options] ... | |
%prog [options] -r <requirements file> [package-index-options] ... | |
%prog [options] [-e] <vcs project url> ... | |
%prog [options] [-e] <local project path> ... | |
%prog [options] <archive url/path> ...""" | |
def __init__(self, *args, **kw): | |
super(InstallCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmd_opts.add_option(cmdoptions.requirements()) | |
cmd_opts.add_option(cmdoptions.constraints()) | |
cmd_opts.add_option(cmdoptions.no_deps()) | |
cmd_opts.add_option(cmdoptions.pre()) | |
cmd_opts.add_option(cmdoptions.editable()) | |
cmd_opts.add_option( | |
'-t', '--target', | |
dest='target_dir', | |
metavar='dir', | |
default=None, | |
help='Install packages into <dir>. ' | |
'By default this will not replace existing files/folders in ' | |
'<dir>. Use --upgrade to replace existing packages in <dir> ' | |
'with new versions.' | |
) | |
cmdoptions.add_target_python_options(cmd_opts) | |
cmd_opts.add_option( | |
'--user', | |
dest='use_user_site', | |
action='store_true', | |
help="Install to the Python user install directory for your " | |
"platform. Typically ~/.local/, or %APPDATA%\\Python on " | |
"Windows. (See the Python documentation for site.USER_BASE " | |
"for full details.) On Debian systems, this is the " | |
"default when running outside of a virtual environment " | |
"and not as root.") | |
cmd_opts.add_option( | |
'--no-user', | |
dest='use_system_location', | |
action='store_true', | |
help=SUPPRESS_HELP) | |
cmd_opts.add_option( | |
'--root', | |
dest='root_path', | |
metavar='dir', | |
default=None, | |
help="Install everything relative to this alternate root " | |
"directory.") | |
cmd_opts.add_option( | |
'--prefix', | |
dest='prefix_path', | |
metavar='dir', | |
default=None, | |
help="Installation prefix where lib, bin and other top-level " | |
"folders are placed") | |
cmd_opts.add_option( | |
'--system', | |
dest='use_system_location', | |
action='store_true', | |
help="Install using the system scheme (overrides --user on " | |
"Debian systems)") | |
cmd_opts.add_option(cmdoptions.build_dir()) | |
cmd_opts.add_option(cmdoptions.src()) | |
cmd_opts.add_option( | |
'-U', '--upgrade', | |
dest='upgrade', | |
action='store_true', | |
help='Upgrade all specified packages to the newest available ' | |
'version. The handling of dependencies depends on the ' | |
'upgrade-strategy used.' | |
) | |
cmd_opts.add_option( | |
'--upgrade-strategy', | |
dest='upgrade_strategy', | |
default='only-if-needed', | |
choices=['only-if-needed', 'eager'], | |
help='Determines how dependency upgrading should be handled ' | |
'[default: %default]. ' | |
'"eager" - dependencies are upgraded regardless of ' | |
'whether the currently installed version satisfies the ' | |
'requirements of the upgraded package(s). ' | |
'"only-if-needed" - are upgraded only when they do not ' | |
'satisfy the requirements of the upgraded package(s).' | |
) | |
cmd_opts.add_option( | |
'--force-reinstall', | |
dest='force_reinstall', | |
action='store_true', | |
help='Reinstall all packages even if they are already ' | |
'up-to-date.') | |
cmd_opts.add_option( | |
'-I', '--ignore-installed', | |
dest='ignore_installed', | |
action='store_true', | |
help='Ignore the installed packages, overwriting them. ' | |
'This can break your system if the existing package ' | |
'is of a different version or was installed ' | |
'with a different package manager!' | |
) | |
cmd_opts.add_option(cmdoptions.ignore_requires_python()) | |
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |
cmd_opts.add_option(cmdoptions.use_pep517()) | |
cmd_opts.add_option(cmdoptions.no_use_pep517()) | |
cmd_opts.add_option(cmdoptions.install_options()) | |
cmd_opts.add_option(cmdoptions.global_options()) | |
cmd_opts.add_option( | |
"--compile", | |
action="store_true", | |
dest="compile", | |
default=True, | |
help="Compile Python source files to bytecode", | |
) | |
cmd_opts.add_option( | |
"--no-compile", | |
action="store_false", | |
dest="compile", | |
help="Do not compile Python source files to bytecode", | |
) | |
cmd_opts.add_option( | |
"--no-warn-script-location", | |
action="store_false", | |
dest="warn_script_location", | |
default=True, | |
help="Do not warn when installing scripts outside PATH", | |
) | |
cmd_opts.add_option( | |
"--no-warn-conflicts", | |
action="store_false", | |
dest="warn_about_conflicts", | |
default=True, | |
help="Do not warn about broken dependencies", | |
) | |
cmd_opts.add_option(cmdoptions.no_binary()) | |
cmd_opts.add_option(cmdoptions.only_binary()) | |
cmd_opts.add_option(cmdoptions.prefer_binary()) | |
cmd_opts.add_option(cmdoptions.no_clean()) | |
cmd_opts.add_option(cmdoptions.require_hashes()) | |
cmd_opts.add_option(cmdoptions.progress_bar()) | |
index_opts = cmdoptions.make_option_group( | |
cmdoptions.index_group, | |
self.parser, | |
) | |
self.parser.insert_option_group(0, index_opts) | |
self.parser.insert_option_group(0, cmd_opts) | |
def run(self, options, args): | |
# type: (Values, List[Any]) -> int | |
cmdoptions.check_install_build_global(options) | |
upgrade_strategy = "to-satisfy-only" | |
if options.upgrade: | |
upgrade_strategy = options.upgrade_strategy | |
cmdoptions.check_dist_restriction(options, check_target=True) | |
if options.python_version: | |
python_versions = [options.python_version] | |
else: | |
python_versions = None | |
# compute install location defaults | |
if (not options.use_user_site and not options.prefix_path and not | |
options.target_dir and not options.use_system_location): | |
if not running_under_virtualenv() and os.geteuid() != 0: | |
options.use_user_site = True | |
if options.use_system_location: | |
options.use_user_site = False | |
options.src_dir = os.path.abspath(options.src_dir) | |
install_options = options.install_options or [] | |
options.use_user_site = decide_user_install( | |
options.use_user_site, | |
prefix_path=options.prefix_path, | |
target_dir=options.target_dir, | |
root_path=options.root_path, | |
isolated_mode=options.isolated_mode, | |
) | |
target_temp_dir = None # type: Optional[TempDirectory] | |
target_temp_dir_path = None # type: Optional[str] | |
if options.target_dir: | |
options.ignore_installed = True | |
options.target_dir = os.path.abspath(options.target_dir) | |
if (os.path.exists(options.target_dir) and not | |
os.path.isdir(options.target_dir)): | |
raise CommandError( | |
"Target path exists but is not a directory, will not " | |
"continue." | |
) | |
# Create a target directory for using with the target option | |
target_temp_dir = TempDirectory(kind="target") | |
target_temp_dir_path = target_temp_dir.path | |
global_options = options.global_options or [] | |
session = self.get_default_session(options) | |
target_python = make_target_python(options) | |
finder = self._build_package_finder( | |
options=options, | |
session=session, | |
target_python=target_python, | |
ignore_requires_python=options.ignore_requires_python, | |
) | |
build_delete = (not (options.no_clean or options.build_dir)) | |
wheel_cache = WheelCache(options.cache_dir, options.format_control) | |
with get_requirement_tracker() as req_tracker, TempDirectory( | |
options.build_dir, delete=build_delete, kind="install" | |
) as directory: | |
requirement_set = RequirementSet( | |
check_supported_wheels=not options.target_dir, | |
) | |
try: | |
self.populate_requirement_set( | |
requirement_set, args, options, finder, session, | |
wheel_cache | |
) | |
warn_deprecated_install_options( | |
requirement_set, options.install_options | |
) | |
preparer = self.make_requirement_preparer( | |
temp_build_dir=directory, | |
options=options, | |
req_tracker=req_tracker, | |
session=session, | |
finder=finder, | |
use_user_site=options.use_user_site, | |
) | |
resolver = self.make_resolver( | |
preparer=preparer, | |
finder=finder, | |
options=options, | |
wheel_cache=wheel_cache, | |
use_user_site=options.use_user_site, | |
ignore_installed=options.ignore_installed, | |
ignore_requires_python=options.ignore_requires_python, | |
force_reinstall=options.force_reinstall, | |
upgrade_strategy=upgrade_strategy, | |
use_pep517=options.use_pep517, | |
) | |
self.trace_basic_info(finder) | |
resolver.resolve(requirement_set) | |
try: | |
pip_req = requirement_set.get_requirement("pip") | |
except KeyError: | |
modifying_pip = None | |
else: | |
# If we're not replacing an already installed pip, | |
# we're not modifying it. | |
modifying_pip = pip_req.satisfied_by is None | |
protect_pip_from_modification_on_windows( | |
modifying_pip=modifying_pip | |
) | |
check_binary_allowed = get_check_binary_allowed( | |
finder.format_control | |
) | |
reqs_to_build = [ | |
r for r in requirement_set.requirements.values() | |
if should_build_for_install_command( | |
r, check_binary_allowed | |
) | |
] | |
_, build_failures = build( | |
reqs_to_build, | |
wheel_cache=wheel_cache, | |
build_options=[], | |
global_options=[], | |
) | |
# If we're using PEP 517, we cannot do a direct install | |
# so we fail here. | |
# We don't care about failures building legacy | |
# requirements, as we'll fall through to a direct | |
# install for those. | |
pep517_build_failures = [ | |
r for r in build_failures if r.use_pep517 | |
] | |
if pep517_build_failures: | |
raise InstallationError( | |
"Could not build wheels for {} which use" | |
" PEP 517 and cannot be installed directly".format( | |
", ".join(r.name for r in pep517_build_failures))) | |
to_install = resolver.get_installation_order( | |
requirement_set | |
) | |
# Consistency Checking of the package set we're installing. | |
should_warn_about_conflicts = ( | |
not options.ignore_dependencies and | |
options.warn_about_conflicts | |
) | |
if should_warn_about_conflicts: | |
self._warn_about_conflicts(to_install) | |
# Don't warn about script install locations if | |
# --target has been specified | |
warn_script_location = options.warn_script_location | |
if options.target_dir: | |
warn_script_location = False | |
installed = install_given_reqs( | |
to_install, | |
install_options, | |
global_options, | |
root=options.root_path, | |
home=target_temp_dir_path, | |
prefix=options.prefix_path, | |
pycompile=options.compile, | |
warn_script_location=warn_script_location, | |
use_user_site=options.use_user_site, | |
) | |
lib_locations = get_lib_location_guesses( | |
user=options.use_user_site, | |
home=target_temp_dir_path, | |
root=options.root_path, | |
prefix=options.prefix_path, | |
isolated=options.isolated_mode, | |
) | |
working_set = pkg_resources.WorkingSet(lib_locations) | |
installed.sort(key=operator.attrgetter('name')) | |
items = [] | |
for result in installed: | |
item = result.name | |
try: | |
installed_version = get_installed_version( | |
result.name, working_set=working_set | |
) | |
if installed_version: | |
item += '-' + installed_version | |
except Exception: | |
pass | |
items.append(item) | |
installed_desc = ' '.join(items) | |
if installed_desc: | |
write_output( | |
'Successfully installed %s', installed_desc, | |
) | |
except EnvironmentError as error: | |
show_traceback = (self.verbosity >= 1) | |
message = create_env_error_message( | |
error, show_traceback, options.use_user_site, | |
) | |
logger.error(message, exc_info=show_traceback) | |
return ERROR | |
except PreviousBuildDirError: | |
options.no_clean = True | |
raise | |
finally: | |
# Clean up | |
if not options.no_clean: | |
requirement_set.cleanup_files() | |
wheel_cache.cleanup() | |
if options.target_dir: | |
self._handle_target_dir( | |
options.target_dir, target_temp_dir, options.upgrade | |
) | |
return SUCCESS | |
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): | |
ensure_dir(target_dir) | |
# Checking both purelib and platlib directories for installed | |
# packages to be moved to target directory | |
lib_dir_list = [] | |
with target_temp_dir: | |
# Checking both purelib and platlib directories for installed | |
# packages to be moved to target directory | |
scheme = distutils_scheme('', home=target_temp_dir.path) | |
purelib_dir = scheme['purelib'] | |
platlib_dir = scheme['platlib'] | |
data_dir = scheme['data'] | |
if os.path.exists(purelib_dir): | |
lib_dir_list.append(purelib_dir) | |
if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: | |
lib_dir_list.append(platlib_dir) | |
if os.path.exists(data_dir): | |
lib_dir_list.append(data_dir) | |
for lib_dir in lib_dir_list: | |
for item in os.listdir(lib_dir): | |
if lib_dir == data_dir: | |
ddir = os.path.join(data_dir, item) | |
if any(s.startswith(ddir) for s in lib_dir_list[:-1]): | |
continue | |
target_item_dir = os.path.join(target_dir, item) | |
if os.path.exists(target_item_dir): | |
if not upgrade: | |
logger.warning( | |
'Target directory %s already exists. Specify ' | |
'--upgrade to force replacement.', | |
target_item_dir | |
) | |
continue | |
if os.path.islink(target_item_dir): | |
logger.warning( | |
'Target directory %s already exists and is ' | |
'a link. Pip will not automatically replace ' | |
'links, please remove if replacement is ' | |
'desired.', | |
target_item_dir | |
) | |
continue | |
if os.path.isdir(target_item_dir): | |
shutil.rmtree(target_item_dir) | |
else: | |
os.remove(target_item_dir) | |
shutil.move( | |
os.path.join(lib_dir, item), | |
target_item_dir | |
) | |
def _warn_about_conflicts(self, to_install): | |
try: | |
package_set, _dep_info = check_install_conflicts(to_install) | |
except Exception: | |
logger.error("Error checking for conflicts.", exc_info=True) | |
return | |
missing, conflicting = _dep_info | |
# NOTE: There is some duplication here from pip check | |
for project_name in missing: | |
version = package_set[project_name][0] | |
for dependency in missing[project_name]: | |
logger.critical( | |
"%s %s requires %s, which is not installed.", | |
project_name, version, dependency[1], | |
) | |
for project_name in conflicting: | |
version = package_set[project_name][0] | |
for dep_name, dep_version, req in conflicting[project_name]: | |
logger.critical( | |
"%s %s has requirement %s, but you'll have %s %s which is " | |
"incompatible.", | |
project_name, version, req, dep_name, dep_version, | |
) | |
def get_lib_location_guesses(*args, **kwargs): | |
scheme = distutils_scheme('', *args, **kwargs) | |
return [scheme['purelib'], scheme['platlib']] | |
def site_packages_writable(**kwargs): | |
return all( | |
test_writable_dir(d) for d in set(get_lib_location_guesses(**kwargs)) | |
) | |
def decide_user_install( | |
use_user_site, # type: Optional[bool] | |
prefix_path=None, # type: Optional[str] | |
target_dir=None, # type: Optional[str] | |
root_path=None, # type: Optional[str] | |
isolated_mode=False, # type: bool | |
): | |
# type: (...) -> bool | |
"""Determine whether to do a user install based on the input options. | |
If use_user_site is False, no additional checks are done. | |
If use_user_site is True, it is checked for compatibility with other | |
options. | |
If use_user_site is None, the default behaviour depends on the environment, | |
which is provided by the other arguments. | |
""" | |
# In some cases (config from tox), use_user_site can be set to an integer | |
# rather than a bool, which 'use_user_site is False' wouldn't catch. | |
if (use_user_site is not None) and (not use_user_site): | |
logger.debug("Non-user install by explicit request") | |
return False | |
if use_user_site: | |
if prefix_path: | |
raise CommandError( | |
"Can not combine '--user' and '--prefix' as they imply " | |
"different installation locations" | |
) | |
if virtualenv_no_global(): | |
raise InstallationError( | |
"Can not perform a '--user' install. User site-packages " | |
"are not visible in this virtualenv." | |
) | |
logger.debug("User install by explicit request") | |
return True | |
# If we are here, user installs have not been explicitly requested/avoided | |
assert use_user_site is None | |
# user install incompatible with --prefix/--target | |
if prefix_path or target_dir: | |
logger.debug("Non-user install due to --prefix or --target option") | |
return False | |
# If user installs are not enabled, choose a non-user install | |
if not site.ENABLE_USER_SITE: | |
logger.debug("Non-user install because user site-packages disabled") | |
return False | |
# If we have permission for a non-user install, do that, | |
# otherwise do a user install. | |
if site_packages_writable(root=root_path, isolated=isolated_mode): | |
logger.debug("Non-user install because site-packages writeable") | |
return False | |
logger.info("Defaulting to user installation because normal site-packages " | |
"is not writeable") | |
return True | |
def warn_deprecated_install_options(requirement_set, options): | |
# type: (RequirementSet, Optional[List[str]]) -> None | |
"""If any location-changing --install-option arguments were passed for | |
requirements or on the command-line, then show a deprecation warning. | |
""" | |
def format_options(option_names): | |
# type: (Iterable[str]) -> List[str] | |
return ["--{}".format(name.replace("_", "-")) for name in option_names] | |
requirements = ( | |
requirement_set.unnamed_requirements + | |
list(requirement_set.requirements.values()) | |
) | |
offenders = [] | |
for requirement in requirements: | |
install_options = requirement.options.get("install_options", []) | |
location_options = parse_distutils_args(install_options) | |
if location_options: | |
offenders.append( | |
"{!r} from {}".format( | |
format_options(location_options.keys()), requirement | |
) | |
) | |
if options: | |
location_options = parse_distutils_args(options) | |
if location_options: | |
offenders.append( | |
"{!r} from command line".format( | |
format_options(location_options.keys()) | |
) | |
) | |
if not offenders: | |
return | |
deprecated( | |
reason=( | |
"Location-changing options found in --install-option: {}. " | |
"This configuration may cause unexpected behavior and is " | |
"unsupported.".format( | |
"; ".join(offenders) | |
) | |
), | |
replacement=( | |
"using pip-level options like --user, --prefix, --root, and " | |
"--target" | |
), | |
gone_in="20.2", | |
issue=7309, | |
) | |
def create_env_error_message(error, show_traceback, using_user_site): | |
"""Format an error message for an EnvironmentError | |
It may occur anytime during the execution of the install command. | |
""" | |
parts = [] | |
# Mention the error if we are not going to show a traceback | |
parts.append("Could not install packages due to an EnvironmentError") | |
if not show_traceback: | |
parts.append(": ") | |
parts.append(str(error)) | |
else: | |
parts.append(".") | |
# Spilt the error indication from a helper message (if any) | |
parts[-1] += "\n" | |
# Suggest useful actions to the user: | |
# (1) using user site-packages or (2) verifying the permissions | |
if error.errno == errno.EACCES: | |
user_option_part = "Consider using the `--user` option" | |
permissions_part = "Check the permissions" | |
if not using_user_site: | |
parts.extend([ | |
user_option_part, " or ", | |
permissions_part.lower(), | |
]) | |
else: | |
parts.append(permissions_part) | |
parts.append(".\n") | |
return "".join(parts).strip() + "\n" |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import json | |
import logging | |
from pip._vendor import six | |
from pip._vendor.six.moves import zip_longest | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.req_command import IndexGroupCommand | |
from pip._internal.exceptions import CommandError | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.models.selection_prefs import SelectionPreferences | |
from pip._internal.self_outdated_check import make_link_collector | |
from pip._internal.utils.misc import ( | |
dist_is_editable, | |
get_installed_distributions, | |
write_output, | |
) | |
from pip._internal.utils.packaging import get_installer | |
from pip._vendor.packaging.version import parse | |
logger = logging.getLogger(__name__) | |
class ListCommand(IndexGroupCommand): | |
""" | |
List installed packages, including editables. | |
Packages are listed in a case-insensitive sorted order. | |
""" | |
usage = """ | |
%prog [options]""" | |
def __init__(self, *args, **kw): | |
super(ListCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmd_opts.add_option( | |
'-o', '--outdated', | |
action='store_true', | |
default=False, | |
help='List outdated packages') | |
cmd_opts.add_option( | |
'-u', '--uptodate', | |
action='store_true', | |
default=False, | |
help='List uptodate packages') | |
cmd_opts.add_option( | |
'-e', '--editable', | |
action='store_true', | |
default=False, | |
help='List editable projects.') | |
cmd_opts.add_option( | |
'-l', '--local', | |
action='store_true', | |
default=False, | |
help=('If in a virtualenv that has global access, do not list ' | |
'globally-installed packages.'), | |
) | |
self.cmd_opts.add_option( | |
'--user', | |
dest='user', | |
action='store_true', | |
default=False, | |
help='Only output packages installed in user-site.') | |
cmd_opts.add_option(cmdoptions.list_path()) | |
cmd_opts.add_option( | |
'--pre', | |
action='store_true', | |
default=False, | |
help=("Include pre-release and development versions. By default, " | |
"pip only finds stable versions."), | |
) | |
cmd_opts.add_option( | |
'--format', | |
action='store', | |
dest='list_format', | |
default="columns", | |
choices=('columns', 'freeze', 'json'), | |
help="Select the output format among: columns (default), freeze, " | |
"or json", | |
) | |
cmd_opts.add_option( | |
'--not-required', | |
action='store_true', | |
dest='not_required', | |
help="List packages that are not dependencies of " | |
"installed packages.", | |
) | |
cmd_opts.add_option( | |
'--exclude-editable', | |
action='store_false', | |
dest='include_editable', | |
help='Exclude editable package from output.', | |
) | |
cmd_opts.add_option( | |
'--include-editable', | |
action='store_true', | |
dest='include_editable', | |
help='Include editable package from output.', | |
default=True, | |
) | |
index_opts = cmdoptions.make_option_group( | |
cmdoptions.index_group, self.parser | |
) | |
self.parser.insert_option_group(0, index_opts) | |
self.parser.insert_option_group(0, cmd_opts) | |
def _build_package_finder(self, options, session): | |
""" | |
Create a package finder appropriate to this list command. | |
""" | |
link_collector = make_link_collector(session, options=options) | |
# Pass allow_yanked=False to ignore yanked versions. | |
selection_prefs = SelectionPreferences( | |
allow_yanked=False, | |
allow_all_prereleases=options.pre, | |
) | |
return PackageFinder.create( | |
link_collector=link_collector, | |
selection_prefs=selection_prefs, | |
) | |
def run(self, options, args): | |
if options.outdated and options.uptodate: | |
raise CommandError( | |
"Options --outdated and --uptodate cannot be combined.") | |
cmdoptions.check_list_path_option(options) | |
packages = get_installed_distributions( | |
local_only=options.local, | |
user_only=options.user, | |
editables_only=options.editable, | |
include_editables=options.include_editable, | |
paths=options.path, | |
) | |
# get_not_required must be called firstly in order to find and | |
# filter out all dependencies correctly. Otherwise a package | |
# can't be identified as requirement because some parent packages | |
# could be filtered out before. | |
if options.not_required: | |
packages = self.get_not_required(packages, options) | |
if options.outdated: | |
packages = self.get_outdated(packages, options) | |
elif options.uptodate: | |
packages = self.get_uptodate(packages, options) | |
self.output_package_listing(packages, options) | |
def get_outdated(self, packages, options): | |
return [ | |
dist for dist in self.iter_packages_latest_infos(packages, options) | |
if parse(str(dist.latest_version)) > parse(str(dist.parsed_version)) | |
] | |
def get_uptodate(self, packages, options): | |
return [ | |
dist for dist in self.iter_packages_latest_infos(packages, options) | |
if parse(str(dist.latest_version)) == parse(str(dist.parsed_version)) | |
] | |
def get_not_required(self, packages, options): | |
dep_keys = set() | |
for dist in packages: | |
dep_keys.update(requirement.key for requirement in dist.requires()) | |
return {pkg for pkg in packages if pkg.key not in dep_keys} | |
def iter_packages_latest_infos(self, packages, options): | |
with self._build_session(options) as session: | |
finder = self._build_package_finder(options, session) | |
for dist in packages: | |
typ = 'unknown' | |
all_candidates = finder.find_all_candidates(dist.key) | |
if not options.pre: | |
# Remove prereleases | |
all_candidates = [candidate for candidate in all_candidates | |
if not candidate.version.is_prerelease] | |
evaluator = finder.make_candidate_evaluator( | |
project_name=dist.project_name, | |
) | |
best_candidate = evaluator.sort_best_candidate(all_candidates) | |
if best_candidate is None: | |
continue | |
remote_version = best_candidate.version | |
if best_candidate.link.is_wheel: | |
typ = 'wheel' | |
else: | |
typ = 'sdist' | |
# This is dirty but makes the rest of the code much cleaner | |
dist.latest_version = remote_version | |
dist.latest_filetype = typ | |
yield dist | |
def output_package_listing(self, packages, options): | |
packages = sorted( | |
packages, | |
key=lambda dist: dist.project_name.lower(), | |
) | |
if options.list_format == 'columns' and packages: | |
data, header = format_for_columns(packages, options) | |
self.output_package_listing_columns(data, header) | |
elif options.list_format == 'freeze': | |
for dist in packages: | |
if options.verbose >= 1: | |
write_output("%s==%s (%s)", dist.project_name, | |
dist.version, dist.location) | |
else: | |
write_output("%s==%s", dist.project_name, dist.version) | |
elif options.list_format == 'json': | |
write_output(format_for_json(packages, options)) | |
def output_package_listing_columns(self, data, header): | |
# insert the header first: we need to know the size of column names | |
if len(data) > 0: | |
data.insert(0, header) | |
pkg_strings, sizes = tabulate(data) | |
# Create and add a separator. | |
if len(data) > 0: | |
pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes))) | |
for val in pkg_strings: | |
write_output(val) | |
def tabulate(vals): | |
# From pfmoore on GitHub: | |
# https://github.com/pypa/pip/issues/3651#issuecomment-216932564 | |
assert len(vals) > 0 | |
sizes = [0] * max(len(x) for x in vals) | |
for row in vals: | |
sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)] | |
result = [] | |
for row in vals: | |
display = " ".join([str(c).ljust(s) if c is not None else '' | |
for s, c in zip_longest(sizes, row)]) | |
result.append(display) | |
return result, sizes | |
def format_for_columns(pkgs, options): | |
""" | |
Convert the package data into something usable | |
by output_package_listing_columns. | |
""" | |
running_outdated = options.outdated | |
# Adjust the header for the `pip list --outdated` case. | |
if running_outdated: | |
header = ["Package", "Version", "Latest", "Type"] | |
else: | |
header = ["Package", "Version"] | |
data = [] | |
if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs): | |
header.append("Location") | |
if options.verbose >= 1: | |
header.append("Installer") | |
for proj in pkgs: | |
# if we're working on the 'outdated' list, separate out the | |
# latest_version and type | |
row = [proj.project_name, proj.version] | |
if running_outdated: | |
row.append(proj.latest_version) | |
row.append(proj.latest_filetype) | |
if options.verbose >= 1 or dist_is_editable(proj): | |
row.append(proj.location) | |
if options.verbose >= 1: | |
row.append(get_installer(proj)) | |
data.append(row) | |
return data, header | |
def format_for_json(packages, options): | |
data = [] | |
for dist in packages: | |
info = { | |
'name': dist.project_name, | |
'version': six.text_type(dist.version), | |
} | |
if options.verbose >= 1: | |
info['location'] = dist.location | |
info['installer'] = get_installer(dist) | |
if options.outdated: | |
info['latest_version'] = six.text_type(dist.latest_version) | |
info['latest_filetype'] = dist.latest_filetype | |
data.append(info) | |
return json.dumps(data) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import sys | |
import textwrap | |
from collections import OrderedDict | |
from pip._vendor import pkg_resources | |
from pip._vendor.packaging.version import parse as parse_version | |
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | |
# why we ignore the type on this import | |
from pip._vendor.six.moves import xmlrpc_client # type: ignore | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.req_command import SessionCommandMixin | |
from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS | |
from pip._internal.exceptions import CommandError | |
from pip._internal.models.index import PyPI | |
from pip._internal.network.xmlrpc import PipXmlrpcTransport | |
from pip._internal.utils.compat import get_terminal_size | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import write_output | |
logger = logging.getLogger(__name__) | |
class SearchCommand(Command, SessionCommandMixin): | |
"""Search for PyPI packages whose name or summary contains <query>.""" | |
usage = """ | |
%prog [options] <query>""" | |
ignore_require_venv = True | |
def __init__(self, *args, **kw): | |
super(SearchCommand, self).__init__(*args, **kw) | |
self.cmd_opts.add_option( | |
'-i', '--index', | |
dest='index', | |
metavar='URL', | |
default=PyPI.pypi_url, | |
help='Base URL of Python Package Index (default %default)') | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
if not args: | |
raise CommandError('Missing required argument (search query).') | |
query = args | |
pypi_hits = self.search(query, options) | |
hits = transform_hits(pypi_hits) | |
terminal_width = None | |
if sys.stdout.isatty(): | |
terminal_width = get_terminal_size()[0] | |
print_results(hits, terminal_width=terminal_width) | |
if pypi_hits: | |
return SUCCESS | |
return NO_MATCHES_FOUND | |
def search(self, query, options): | |
index_url = options.index | |
session = self.get_default_session(options) | |
transport = PipXmlrpcTransport(index_url, session) | |
pypi = xmlrpc_client.ServerProxy(index_url, transport) | |
hits = pypi.search({'name': query, 'summary': query}, 'or') | |
return hits | |
def transform_hits(hits): | |
""" | |
The list from pypi is really a list of versions. We want a list of | |
packages with the list of versions stored inline. This converts the | |
list from pypi into one we can use. | |
""" | |
packages = OrderedDict() | |
for hit in hits: | |
name = hit['name'] | |
summary = hit['summary'] | |
version = hit['version'] | |
if name not in packages.keys(): | |
packages[name] = { | |
'name': name, | |
'summary': summary, | |
'versions': [version], | |
} | |
else: | |
packages[name]['versions'].append(version) | |
# if this is the highest version, replace summary and score | |
if version == highest_version(packages[name]['versions']): | |
packages[name]['summary'] = summary | |
return list(packages.values()) | |
def print_results(hits, name_column_width=None, terminal_width=None): | |
if not hits: | |
return | |
if name_column_width is None: | |
name_column_width = max([ | |
len(hit['name']) + len(highest_version(hit.get('versions', ['-']))) | |
for hit in hits | |
]) + 4 | |
installed_packages = [p.project_name for p in pkg_resources.working_set] | |
for hit in hits: | |
name = hit['name'] | |
summary = hit['summary'] or '' | |
latest = highest_version(hit.get('versions', ['-'])) | |
if terminal_width is not None: | |
target_width = terminal_width - name_column_width - 5 | |
if target_width > 10: | |
# wrap and indent summary to fit terminal | |
summary = textwrap.wrap(summary, target_width) | |
summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) | |
line = '%-*s - %s' % (name_column_width, | |
'%s (%s)' % (name, latest), summary) | |
try: | |
write_output(line) | |
if name in installed_packages: | |
dist = pkg_resources.get_distribution(name) | |
with indent_log(): | |
if dist.version == latest: | |
write_output('INSTALLED: %s (latest)', dist.version) | |
else: | |
write_output('INSTALLED: %s', dist.version) | |
if parse_version(latest).pre: | |
write_output('LATEST: %s (pre-release; install' | |
' with "pip install --pre")', latest) | |
else: | |
write_output('LATEST: %s', latest) | |
except UnicodeEncodeError: | |
pass | |
def highest_version(versions): | |
return max(versions, key=parse_version) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
from email.parser import FeedParser | |
from pip._vendor import pkg_resources | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.status_codes import ERROR, SUCCESS | |
from pip._internal.utils.misc import write_output | |
logger = logging.getLogger(__name__) | |
class ShowCommand(Command): | |
""" | |
Show information about one or more installed packages. | |
The output is in RFC-compliant mail header format. | |
""" | |
usage = """ | |
%prog [options] <package> ...""" | |
ignore_require_venv = True | |
def __init__(self, *args, **kw): | |
super(ShowCommand, self).__init__(*args, **kw) | |
self.cmd_opts.add_option( | |
'-f', '--files', | |
dest='files', | |
action='store_true', | |
default=False, | |
help='Show the full list of installed files for each package.') | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
if not args: | |
logger.warning('ERROR: Please provide a package name or names.') | |
return ERROR | |
query = args | |
results = search_packages_info(query) | |
if not print_results( | |
results, list_files=options.files, verbose=options.verbose): | |
return ERROR | |
return SUCCESS | |
def search_packages_info(query): | |
""" | |
Gather details from installed distributions. Print distribution name, | |
version, location, and installed files. Installed files requires a | |
pip generated 'installed-files.txt' in the distributions '.egg-info' | |
directory. | |
""" | |
installed = {} | |
for p in pkg_resources.working_set: | |
installed[canonicalize_name(p.project_name)] = p | |
query_names = [canonicalize_name(name) for name in query] | |
missing = sorted( | |
[name for name, pkg in zip(query, query_names) if pkg not in installed] | |
) | |
if missing: | |
logger.warning('Package(s) not found: %s', ', '.join(missing)) | |
def get_requiring_packages(package_name): | |
canonical_name = canonicalize_name(package_name) | |
return [ | |
pkg.project_name for pkg in pkg_resources.working_set | |
if canonical_name in | |
[canonicalize_name(required.name) for required in | |
pkg.requires()] | |
] | |
for dist in [installed[pkg] for pkg in query_names if pkg in installed]: | |
package = { | |
'name': dist.project_name, | |
'version': dist.version, | |
'location': dist.location, | |
'requires': [dep.project_name for dep in dist.requires()], | |
'required_by': get_requiring_packages(dist.project_name) | |
} | |
file_list = None | |
metadata = None | |
if isinstance(dist, pkg_resources.DistInfoDistribution): | |
# RECORDs should be part of .dist-info metadatas | |
if dist.has_metadata('RECORD'): | |
lines = dist.get_metadata_lines('RECORD') | |
paths = [l.split(',')[0] for l in lines] | |
paths = [os.path.join(dist.location, p) for p in paths] | |
file_list = [os.path.relpath(p, dist.location) for p in paths] | |
if dist.has_metadata('METADATA'): | |
metadata = dist.get_metadata('METADATA') | |
else: | |
# Otherwise use pip's log for .egg-info's | |
if dist.has_metadata('installed-files.txt'): | |
paths = dist.get_metadata_lines('installed-files.txt') | |
paths = [os.path.join(dist.egg_info, p) for p in paths] | |
file_list = [os.path.relpath(p, dist.location) for p in paths] | |
if dist.has_metadata('PKG-INFO'): | |
metadata = dist.get_metadata('PKG-INFO') | |
if dist.has_metadata('entry_points.txt'): | |
entry_points = dist.get_metadata_lines('entry_points.txt') | |
package['entry_points'] = entry_points | |
if dist.has_metadata('INSTALLER'): | |
for line in dist.get_metadata_lines('INSTALLER'): | |
if line.strip(): | |
package['installer'] = line.strip() | |
break | |
# @todo: Should pkg_resources.Distribution have a | |
# `get_pkg_info` method? | |
feed_parser = FeedParser() | |
feed_parser.feed(metadata) | |
pkg_info_dict = feed_parser.close() | |
for key in ('metadata-version', 'summary', | |
'home-page', 'author', 'author-email', 'license'): | |
package[key] = pkg_info_dict.get(key) | |
# It looks like FeedParser cannot deal with repeated headers | |
classifiers = [] | |
for line in metadata.splitlines(): | |
if line.startswith('Classifier: '): | |
classifiers.append(line[len('Classifier: '):]) | |
package['classifiers'] = classifiers | |
if file_list: | |
package['files'] = sorted(file_list) | |
yield package | |
def print_results(distributions, list_files=False, verbose=False): | |
""" | |
Print the informations from installed distributions found. | |
""" | |
results_printed = False | |
for i, dist in enumerate(distributions): | |
results_printed = True | |
if i > 0: | |
write_output("---") | |
write_output("Name: %s", dist.get('name', '')) | |
write_output("Version: %s", dist.get('version', '')) | |
write_output("Summary: %s", dist.get('summary', '')) | |
write_output("Home-page: %s", dist.get('home-page', '')) | |
write_output("Author: %s", dist.get('author', '')) | |
write_output("Author-email: %s", dist.get('author-email', '')) | |
write_output("License: %s", dist.get('license', '')) | |
write_output("Location: %s", dist.get('location', '')) | |
write_output("Requires: %s", ', '.join(dist.get('requires', []))) | |
write_output("Required-by: %s", ', '.join(dist.get('required_by', []))) | |
if verbose: | |
write_output("Metadata-Version: %s", | |
dist.get('metadata-version', '')) | |
write_output("Installer: %s", dist.get('installer', '')) | |
write_output("Classifiers:") | |
for classifier in dist.get('classifiers', []): | |
write_output(" %s", classifier) | |
write_output("Entry-points:") | |
for entry in dist.get('entry_points', []): | |
write_output(" %s", entry.strip()) | |
if list_files: | |
write_output("Files:") | |
for line in dist.get('files', []): | |
write_output(" %s", line.strip()) | |
if "files" not in dist: | |
write_output("Cannot locate installed-files.txt") | |
return results_printed |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal.cli.base_command import Command | |
from pip._internal.cli.req_command import SessionCommandMixin | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.req import parse_requirements | |
from pip._internal.req.constructors import install_req_from_line | |
from pip._internal.utils.misc import protect_pip_from_modification_on_windows | |
class UninstallCommand(Command, SessionCommandMixin): | |
""" | |
Uninstall packages. | |
pip is able to uninstall most installed packages. Known exceptions are: | |
- Pure distutils packages installed with ``python setup.py install``, which | |
leave behind no metadata to determine what files were installed. | |
- Script wrappers installed by ``python setup.py develop``. | |
""" | |
usage = """ | |
%prog [options] <package> ... | |
%prog [options] -r <requirements file> ...""" | |
def __init__(self, *args, **kw): | |
super(UninstallCommand, self).__init__(*args, **kw) | |
self.cmd_opts.add_option( | |
'-r', '--requirement', | |
dest='requirements', | |
action='append', | |
default=[], | |
metavar='file', | |
help='Uninstall all the packages listed in the given requirements ' | |
'file. This option can be used multiple times.', | |
) | |
self.cmd_opts.add_option( | |
'-y', '--yes', | |
dest='yes', | |
action='store_true', | |
help="Don't ask for confirmation of uninstall deletions.") | |
self.parser.insert_option_group(0, self.cmd_opts) | |
def run(self, options, args): | |
session = self.get_default_session(options) | |
reqs_to_uninstall = {} | |
for name in args: | |
req = install_req_from_line( | |
name, isolated=options.isolated_mode, | |
) | |
if req.name: | |
reqs_to_uninstall[canonicalize_name(req.name)] = req | |
for filename in options.requirements: | |
for req in parse_requirements( | |
filename, | |
options=options, | |
session=session): | |
if req.name: | |
reqs_to_uninstall[canonicalize_name(req.name)] = req | |
if not reqs_to_uninstall: | |
raise InstallationError( | |
'You must give at least one requirement to %(name)s (see ' | |
'"pip help %(name)s")' % dict(name=self.name) | |
) | |
protect_pip_from_modification_on_windows( | |
modifying_pip="pip" in reqs_to_uninstall | |
) | |
for req in reqs_to_uninstall.values(): | |
uninstall_pathset = req.uninstall( | |
auto_confirm=options.yes, verbose=self.verbosity > 0, | |
) | |
if uninstall_pathset: | |
uninstall_pathset.commit() |
# -*- coding: utf-8 -*- | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import shutil | |
from pip._internal.cache import WheelCache | |
from pip._internal.cli import cmdoptions | |
from pip._internal.cli.req_command import RequirementCommand | |
from pip._internal.exceptions import CommandError, PreviousBuildDirError | |
from pip._internal.req import RequirementSet | |
from pip._internal.req.req_tracker import get_requirement_tracker | |
from pip._internal.utils.misc import ensure_dir, normalize_path | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.wheel_builder import build, should_build_for_wheel_command | |
if MYPY_CHECK_RUNNING: | |
from optparse import Values | |
from typing import Any, List | |
logger = logging.getLogger(__name__) | |
class WheelCommand(RequirementCommand): | |
""" | |
Build Wheel archives for your requirements and dependencies. | |
Wheel is a built-package format, and offers the advantage of not | |
recompiling your software during every install. For more details, see the | |
wheel docs: https://wheel.readthedocs.io/en/latest/ | |
Requirements: setuptools>=0.8, and wheel. | |
'pip wheel' uses the bdist_wheel setuptools extension from the wheel | |
package to build individual wheels. | |
""" | |
usage = """ | |
%prog [options] <requirement specifier> ... | |
%prog [options] -r <requirements file> ... | |
%prog [options] [-e] <vcs project url> ... | |
%prog [options] [-e] <local project path> ... | |
%prog [options] <archive url/path> ...""" | |
def __init__(self, *args, **kw): | |
super(WheelCommand, self).__init__(*args, **kw) | |
cmd_opts = self.cmd_opts | |
cmd_opts.add_option( | |
'-w', '--wheel-dir', | |
dest='wheel_dir', | |
metavar='dir', | |
default=os.curdir, | |
help=("Build wheels into <dir>, where the default is the " | |
"current working directory."), | |
) | |
cmd_opts.add_option(cmdoptions.no_binary()) | |
cmd_opts.add_option(cmdoptions.only_binary()) | |
cmd_opts.add_option(cmdoptions.prefer_binary()) | |
cmd_opts.add_option( | |
'--build-option', | |
dest='build_options', | |
metavar='options', | |
action='append', | |
help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", | |
) | |
cmd_opts.add_option(cmdoptions.no_build_isolation()) | |
cmd_opts.add_option(cmdoptions.use_pep517()) | |
cmd_opts.add_option(cmdoptions.no_use_pep517()) | |
cmd_opts.add_option(cmdoptions.constraints()) | |
cmd_opts.add_option(cmdoptions.editable()) | |
cmd_opts.add_option(cmdoptions.requirements()) | |
cmd_opts.add_option(cmdoptions.src()) | |
cmd_opts.add_option(cmdoptions.ignore_requires_python()) | |
cmd_opts.add_option(cmdoptions.no_deps()) | |
cmd_opts.add_option(cmdoptions.build_dir()) | |
cmd_opts.add_option(cmdoptions.progress_bar()) | |
cmd_opts.add_option( | |
'--global-option', | |
dest='global_options', | |
action='append', | |
metavar='options', | |
help="Extra global options to be supplied to the setup.py " | |
"call before the 'bdist_wheel' command.") | |
cmd_opts.add_option( | |
'--pre', | |
action='store_true', | |
default=False, | |
help=("Include pre-release and development versions. By default, " | |
"pip only finds stable versions."), | |
) | |
cmd_opts.add_option(cmdoptions.no_clean()) | |
cmd_opts.add_option(cmdoptions.require_hashes()) | |
index_opts = cmdoptions.make_option_group( | |
cmdoptions.index_group, | |
self.parser, | |
) | |
self.parser.insert_option_group(0, index_opts) | |
self.parser.insert_option_group(0, cmd_opts) | |
def run(self, options, args): | |
# type: (Values, List[Any]) -> None | |
cmdoptions.check_install_build_global(options) | |
session = self.get_default_session(options) | |
finder = self._build_package_finder(options, session) | |
build_delete = (not (options.no_clean or options.build_dir)) | |
wheel_cache = WheelCache(options.cache_dir, options.format_control) | |
options.wheel_dir = normalize_path(options.wheel_dir) | |
ensure_dir(options.wheel_dir) | |
with get_requirement_tracker() as req_tracker, TempDirectory( | |
options.build_dir, delete=build_delete, kind="wheel" | |
) as directory: | |
requirement_set = RequirementSet() | |
try: | |
self.populate_requirement_set( | |
requirement_set, args, options, finder, session, | |
wheel_cache | |
) | |
preparer = self.make_requirement_preparer( | |
temp_build_dir=directory, | |
options=options, | |
req_tracker=req_tracker, | |
session=session, | |
finder=finder, | |
wheel_download_dir=options.wheel_dir, | |
use_user_site=False, | |
) | |
resolver = self.make_resolver( | |
preparer=preparer, | |
finder=finder, | |
options=options, | |
wheel_cache=wheel_cache, | |
ignore_requires_python=options.ignore_requires_python, | |
use_pep517=options.use_pep517, | |
) | |
self.trace_basic_info(finder) | |
resolver.resolve(requirement_set) | |
reqs_to_build = [ | |
r for r in requirement_set.requirements.values() | |
if should_build_for_wheel_command(r) | |
] | |
# build wheels | |
build_successes, build_failures = build( | |
reqs_to_build, | |
wheel_cache=wheel_cache, | |
build_options=options.build_options or [], | |
global_options=options.global_options or [], | |
) | |
for req in build_successes: | |
assert req.link and req.link.is_wheel | |
assert req.local_file_path | |
# copy from cache to target directory | |
try: | |
shutil.copy(req.local_file_path, options.wheel_dir) | |
except OSError as e: | |
logger.warning( | |
"Building wheel for %s failed: %s", | |
req.name, e, | |
) | |
build_failures.append(req) | |
if len(build_failures) != 0: | |
raise CommandError( | |
"Failed to build one or more wheels" | |
) | |
except PreviousBuildDirError: | |
options.no_clean = True | |
raise | |
finally: | |
if not options.no_clean: | |
requirement_set.cleanup_files() | |
wheel_cache.cleanup() |
"""Configuration management setup | |
Some terminology: | |
- name | |
As written in config files. | |
- value | |
Value associated with a name | |
- key | |
Name combined with it's section (section.name) | |
- variant | |
A single word describing where the configuration key-value pair came from | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import locale | |
import logging | |
import os | |
import sys | |
from pip._vendor.six.moves import configparser | |
from pip._internal.exceptions import ( | |
ConfigurationError, | |
ConfigurationFileCouldNotBeLoaded, | |
) | |
from pip._internal.utils import appdirs | |
from pip._internal.utils.compat import WINDOWS, expanduser | |
from pip._internal.utils.misc import ensure_dir, enum | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Dict, Iterable, List, NewType, Optional, Tuple | |
) | |
RawConfigParser = configparser.RawConfigParser # Shorthand | |
Kind = NewType("Kind", str) | |
logger = logging.getLogger(__name__) | |
# NOTE: Maybe use the optionx attribute to normalize keynames. | |
def _normalize_name(name): | |
# type: (str) -> str | |
"""Make a name consistent regardless of source (environment or file) | |
""" | |
name = name.lower().replace('_', '-') | |
if name.startswith('--'): | |
name = name[2:] # only prefer long opts | |
return name | |
def _disassemble_key(name): | |
# type: (str) -> List[str] | |
if "." not in name: | |
error_message = ( | |
"Key does not contain dot separated section and key. " | |
"Perhaps you wanted to use 'global.{}' instead?" | |
).format(name) | |
raise ConfigurationError(error_message) | |
return name.split(".", 1) | |
# The kinds of configurations there are. | |
kinds = enum( | |
USER="user", # User Specific | |
GLOBAL="global", # System Wide | |
SITE="site", # [Virtual] Environment Specific | |
ENV="env", # from PIP_CONFIG_FILE | |
ENV_VAR="env-var", # from Environment Variables | |
) | |
CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' | |
def get_configuration_files(): | |
# type: () -> Dict[Kind, List[str]] | |
global_config_files = [ | |
os.path.join(path, CONFIG_BASENAME) | |
for path in appdirs.site_config_dirs('pip') | |
] | |
site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME) | |
legacy_config_file = os.path.join( | |
expanduser('~'), | |
'pip' if WINDOWS else '.pip', | |
CONFIG_BASENAME, | |
) | |
new_config_file = os.path.join( | |
appdirs.user_config_dir("pip"), CONFIG_BASENAME | |
) | |
return { | |
kinds.GLOBAL: global_config_files, | |
kinds.SITE: [site_config_file], | |
kinds.USER: [legacy_config_file, new_config_file], | |
} | |
class Configuration(object): | |
"""Handles management of configuration. | |
Provides an interface to accessing and managing configuration files. | |
This class converts provides an API that takes "section.key-name" style | |
keys and stores the value associated with it as "key-name" under the | |
section "section". | |
This allows for a clean interface wherein the both the section and the | |
key-name are preserved in an easy to manage form in the configuration files | |
and the data stored is also nice. | |
""" | |
def __init__(self, isolated, load_only=None): | |
# type: (bool, Kind) -> None | |
super(Configuration, self).__init__() | |
_valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] | |
if load_only not in _valid_load_only: | |
raise ConfigurationError( | |
"Got invalid value for load_only - should be one of {}".format( | |
", ".join(map(repr, _valid_load_only[:-1])) | |
) | |
) | |
self.isolated = isolated # type: bool | |
self.load_only = load_only # type: Optional[Kind] | |
# The order here determines the override order. | |
self._override_order = [ | |
kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR | |
] | |
self._ignore_env_names = ["version", "help"] | |
# Because we keep track of where we got the data from | |
self._parsers = { | |
variant: [] for variant in self._override_order | |
} # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] | |
self._config = { | |
variant: {} for variant in self._override_order | |
} # type: Dict[Kind, Dict[str, Any]] | |
self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] | |
def load(self): | |
# type: () -> None | |
"""Loads configuration from configuration files and environment | |
""" | |
self._load_config_files() | |
if not self.isolated: | |
self._load_environment_vars() | |
def get_file_to_edit(self): | |
# type: () -> Optional[str] | |
"""Returns the file with highest priority in configuration | |
""" | |
assert self.load_only is not None, \ | |
"Need to be specified a file to be editing" | |
try: | |
return self._get_parser_to_modify()[0] | |
except IndexError: | |
return None | |
def items(self): | |
# type: () -> Iterable[Tuple[str, Any]] | |
"""Returns key-value pairs like dict.items() representing the loaded | |
configuration | |
""" | |
return self._dictionary.items() | |
def get_value(self, key): | |
# type: (str) -> Any | |
"""Get a value from the configuration. | |
""" | |
try: | |
return self._dictionary[key] | |
except KeyError: | |
raise ConfigurationError("No such key - {}".format(key)) | |
def set_value(self, key, value): | |
# type: (str, Any) -> None | |
"""Modify a value in the configuration. | |
""" | |
self._ensure_have_load_only() | |
fname, parser = self._get_parser_to_modify() | |
if parser is not None: | |
section, name = _disassemble_key(key) | |
# Modify the parser and the configuration | |
if not parser.has_section(section): | |
parser.add_section(section) | |
parser.set(section, name, value) | |
self._config[self.load_only][key] = value | |
self._mark_as_modified(fname, parser) | |
def unset_value(self, key): | |
# type: (str) -> None | |
"""Unset a value in the configuration. | |
""" | |
self._ensure_have_load_only() | |
if key not in self._config[self.load_only]: | |
raise ConfigurationError("No such key - {}".format(key)) | |
fname, parser = self._get_parser_to_modify() | |
if parser is not None: | |
section, name = _disassemble_key(key) | |
# Remove the key in the parser | |
modified_something = False | |
if parser.has_section(section): | |
# Returns whether the option was removed or not | |
modified_something = parser.remove_option(section, name) | |
if modified_something: | |
# name removed from parser, section may now be empty | |
section_iter = iter(parser.items(section)) | |
try: | |
val = next(section_iter) | |
except StopIteration: | |
val = None | |
if val is None: | |
parser.remove_section(section) | |
self._mark_as_modified(fname, parser) | |
else: | |
raise ConfigurationError( | |
"Fatal Internal error [id=1]. Please report as a bug." | |
) | |
del self._config[self.load_only][key] | |
def save(self): | |
# type: () -> None | |
"""Save the current in-memory state. | |
""" | |
self._ensure_have_load_only() | |
for fname, parser in self._modified_parsers: | |
logger.info("Writing to %s", fname) | |
# Ensure directory exists. | |
ensure_dir(os.path.dirname(fname)) | |
with open(fname, "w") as f: | |
parser.write(f) | |
# | |
# Private routines | |
# | |
def _ensure_have_load_only(self): | |
# type: () -> None | |
if self.load_only is None: | |
raise ConfigurationError("Needed a specific file to be modifying.") | |
logger.debug("Will be working with %s variant only", self.load_only) | |
@property | |
def _dictionary(self): | |
# type: () -> Dict[str, Any] | |
"""A dictionary representing the loaded configuration. | |
""" | |
# NOTE: Dictionaries are not populated if not loaded. So, conditionals | |
# are not needed here. | |
retval = {} | |
for variant in self._override_order: | |
retval.update(self._config[variant]) | |
return retval | |
def _load_config_files(self): | |
# type: () -> None | |
"""Loads configuration from configuration files | |
""" | |
config_files = dict(self._iter_config_files()) | |
if config_files[kinds.ENV][0:1] == [os.devnull]: | |
logger.debug( | |
"Skipping loading configuration files due to " | |
"environment's PIP_CONFIG_FILE being os.devnull" | |
) | |
return | |
for variant, files in config_files.items(): | |
for fname in files: | |
# If there's specific variant set in `load_only`, load only | |
# that variant, not the others. | |
if self.load_only is not None and variant != self.load_only: | |
logger.debug( | |
"Skipping file '%s' (variant: %s)", fname, variant | |
) | |
continue | |
parser = self._load_file(variant, fname) | |
# Keeping track of the parsers used | |
self._parsers[variant].append((fname, parser)) | |
def _load_file(self, variant, fname): | |
# type: (Kind, str) -> RawConfigParser | |
logger.debug("For variant '%s', will try loading '%s'", variant, fname) | |
parser = self._construct_parser(fname) | |
for section in parser.sections(): | |
items = parser.items(section) | |
self._config[variant].update(self._normalized_keys(section, items)) | |
return parser | |
def _construct_parser(self, fname): | |
# type: (str) -> RawConfigParser | |
parser = configparser.RawConfigParser() | |
# If there is no such file, don't bother reading it but create the | |
# parser anyway, to hold the data. | |
# Doing this is useful when modifying and saving files, where we don't | |
# need to construct a parser. | |
if os.path.exists(fname): | |
try: | |
parser.read(fname) | |
except UnicodeDecodeError: | |
# See https://github.com/pypa/pip/issues/4963 | |
raise ConfigurationFileCouldNotBeLoaded( | |
reason="contains invalid {} characters".format( | |
locale.getpreferredencoding(False) | |
), | |
fname=fname, | |
) | |
except configparser.Error as error: | |
# See https://github.com/pypa/pip/issues/4893 | |
raise ConfigurationFileCouldNotBeLoaded(error=error) | |
return parser | |
def _load_environment_vars(self): | |
# type: () -> None | |
"""Loads configuration from environment variables | |
""" | |
self._config[kinds.ENV_VAR].update( | |
self._normalized_keys(":env:", self._get_environ_vars()) | |
) | |
def _normalized_keys(self, section, items): | |
# type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any] | |
"""Normalizes items to construct a dictionary with normalized keys. | |
This routine is where the names become keys and are made the same | |
regardless of source - configuration files or environment. | |
""" | |
normalized = {} | |
for name, val in items: | |
key = section + "." + _normalize_name(name) | |
normalized[key] = val | |
return normalized | |
def _get_environ_vars(self): | |
# type: () -> Iterable[Tuple[str, str]] | |
"""Returns a generator with all environmental vars with prefix PIP_""" | |
for key, val in os.environ.items(): | |
should_be_yielded = ( | |
key.startswith("PIP_") and | |
key[4:].lower() not in self._ignore_env_names | |
) | |
if should_be_yielded: | |
yield key[4:].lower(), val | |
# XXX: This is patched in the tests. | |
def _iter_config_files(self): | |
# type: () -> Iterable[Tuple[Kind, List[str]]] | |
"""Yields variant and configuration files associated with it. | |
This should be treated like items of a dictionary. | |
""" | |
# SMELL: Move the conditions out of this function | |
# environment variables have the lowest priority | |
config_file = os.environ.get('PIP_CONFIG_FILE', None) | |
if config_file is not None: | |
yield kinds.ENV, [config_file] | |
else: | |
yield kinds.ENV, [] | |
config_files = get_configuration_files() | |
# at the base we have any global configuration | |
yield kinds.GLOBAL, config_files[kinds.GLOBAL] | |
# per-user configuration next | |
should_load_user_config = not self.isolated and not ( | |
config_file and os.path.exists(config_file) | |
) | |
if should_load_user_config: | |
# The legacy config file is overridden by the new config file | |
yield kinds.USER, config_files[kinds.USER] | |
# finally virtualenv configuration first trumping others | |
yield kinds.SITE, config_files[kinds.SITE] | |
def _get_parser_to_modify(self): | |
# type: () -> Tuple[str, RawConfigParser] | |
# Determine which parser to modify | |
parsers = self._parsers[self.load_only] | |
if not parsers: | |
# This should not happen if everything works correctly. | |
raise ConfigurationError( | |
"Fatal Internal error [id=2]. Please report as a bug." | |
) | |
# Use the highest priority parser. | |
return parsers[-1] | |
# XXX: This is patched in the tests. | |
def _mark_as_modified(self, fname, parser): | |
# type: (str, RawConfigParser) -> None | |
file_parser_tuple = (fname, parser) | |
if file_parser_tuple not in self._modified_parsers: | |
self._modified_parsers.append(file_parser_tuple) |
from pip._internal.distributions.sdist import SourceDistribution | |
from pip._internal.distributions.wheel import WheelDistribution | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from pip._internal.distributions.base import AbstractDistribution | |
from pip._internal.req.req_install import InstallRequirement | |
def make_distribution_for_install_requirement(install_req): | |
# type: (InstallRequirement) -> AbstractDistribution | |
"""Returns a Distribution for the given InstallRequirement | |
""" | |
# Editable requirements will always be source distributions. They use the | |
# legacy logic until we create a modern standard for them. | |
if install_req.editable: | |
return SourceDistribution(install_req) | |
# If it's a wheel, it's a WheelDistribution | |
if install_req.is_wheel: | |
return WheelDistribution(install_req) | |
# Otherwise, a SourceDistribution | |
return SourceDistribution(install_req) |
import abc | |
from pip._vendor.six import add_metaclass | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional | |
from pip._vendor.pkg_resources import Distribution | |
from pip._internal.req import InstallRequirement | |
from pip._internal.index.package_finder import PackageFinder | |
@add_metaclass(abc.ABCMeta) | |
class AbstractDistribution(object): | |
"""A base class for handling installable artifacts. | |
The requirements for anything installable are as follows: | |
- we must be able to determine the requirement name | |
(or we can't correctly handle the non-upgrade case). | |
- for packages with setup requirements, we must also be able | |
to determine their requirements without installing additional | |
packages (for the same reason as run-time dependencies) | |
- we must be able to create a Distribution object exposing the | |
above metadata. | |
""" | |
def __init__(self, req): | |
# type: (InstallRequirement) -> None | |
super(AbstractDistribution, self).__init__() | |
self.req = req | |
@abc.abstractmethod | |
def get_pkg_resources_distribution(self): | |
# type: () -> Optional[Distribution] | |
raise NotImplementedError() | |
@abc.abstractmethod | |
def prepare_distribution_metadata(self, finder, build_isolation): | |
# type: (PackageFinder, bool) -> None | |
raise NotImplementedError() |
from pip._internal.distributions.base import AbstractDistribution | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional | |
from pip._vendor.pkg_resources import Distribution | |
from pip._internal.index.package_finder import PackageFinder | |
class InstalledDistribution(AbstractDistribution): | |
"""Represents an installed package. | |
This does not need any preparation as the required information has already | |
been computed. | |
""" | |
def get_pkg_resources_distribution(self): | |
# type: () -> Optional[Distribution] | |
return self.req.satisfied_by | |
def prepare_distribution_metadata(self, finder, build_isolation): | |
# type: (PackageFinder, bool) -> None | |
pass |
import logging | |
from pip._internal.build_env import BuildEnvironment | |
from pip._internal.distributions.base import AbstractDistribution | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.utils.subprocess import runner_with_spinner_message | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Set, Tuple | |
from pip._vendor.pkg_resources import Distribution | |
from pip._internal.index.package_finder import PackageFinder | |
logger = logging.getLogger(__name__) | |
class SourceDistribution(AbstractDistribution): | |
"""Represents a source distribution. | |
The preparation step for these needs metadata for the packages to be | |
generated, either using PEP 517 or using the legacy `setup.py egg_info`. | |
""" | |
def get_pkg_resources_distribution(self): | |
# type: () -> Distribution | |
return self.req.get_dist() | |
def prepare_distribution_metadata(self, finder, build_isolation): | |
# type: (PackageFinder, bool) -> None | |
# Load pyproject.toml, to determine whether PEP 517 is to be used | |
self.req.load_pyproject_toml() | |
# Set up the build isolation, if this requirement should be isolated | |
should_isolate = self.req.use_pep517 and build_isolation | |
if should_isolate: | |
self._setup_isolation(finder) | |
self.req.prepare_metadata() | |
def _setup_isolation(self, finder): | |
# type: (PackageFinder) -> None | |
def _raise_conflicts(conflicting_with, conflicting_reqs): | |
# type: (str, Set[Tuple[str, str]]) -> None | |
format_string = ( | |
"Some build dependencies for {requirement} " | |
"conflict with {conflicting_with}: {description}." | |
) | |
error_message = format_string.format( | |
requirement=self.req, | |
conflicting_with=conflicting_with, | |
description=', '.join( | |
'{} is incompatible with {}'.format(installed, wanted) | |
for installed, wanted in sorted(conflicting) | |
) | |
) | |
raise InstallationError(error_message) | |
# Isolate in a BuildEnvironment and install the build-time | |
# requirements. | |
pyproject_requires = self.req.pyproject_requires | |
assert pyproject_requires is not None | |
self.req.build_env = BuildEnvironment() | |
self.req.build_env.install_requirements( | |
finder, pyproject_requires, 'overlay', | |
"Installing build dependencies" | |
) | |
conflicting, missing = self.req.build_env.check_requirements( | |
self.req.requirements_to_check | |
) | |
if conflicting: | |
_raise_conflicts("PEP 517/518 supported requirements", | |
conflicting) | |
if missing: | |
logger.warning( | |
"Missing build requirements in pyproject.toml for %s.", | |
self.req, | |
) | |
logger.warning( | |
"The project does not specify a build backend, and " | |
"pip cannot fall back to setuptools without %s.", | |
" and ".join(map(repr, sorted(missing))) | |
) | |
# Install any extra build dependencies that the backend requests. | |
# This must be done in a second pass, as the pyproject.toml | |
# dependencies must be installed before we can call the backend. | |
with self.req.build_env: | |
runner = runner_with_spinner_message( | |
"Getting requirements to build wheel" | |
) | |
backend = self.req.pep517_backend | |
assert backend is not None | |
with backend.subprocess_runner(runner): | |
reqs = backend.get_requires_for_build_wheel() | |
conflicting, missing = self.req.build_env.check_requirements(reqs) | |
if conflicting: | |
_raise_conflicts("the backend dependencies", conflicting) | |
self.req.build_env.install_requirements( | |
finder, missing, 'normal', | |
"Installing backend dependencies" | |
) |
from zipfile import ZipFile | |
from pip._internal.distributions.base import AbstractDistribution | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel | |
if MYPY_CHECK_RUNNING: | |
from pip._vendor.pkg_resources import Distribution | |
from pip._internal.index.package_finder import PackageFinder | |
class WheelDistribution(AbstractDistribution): | |
"""Represents a wheel distribution. | |
This does not need any preparation as wheels can be directly unpacked. | |
""" | |
def get_pkg_resources_distribution(self): | |
# type: () -> Distribution | |
"""Loads the metadata from the wheel file into memory and returns a | |
Distribution that uses it, not relying on the wheel file or | |
requirement. | |
""" | |
# Set as part of preparation during download. | |
assert self.req.local_file_path | |
# Wheels are never unnamed. | |
assert self.req.name | |
with ZipFile(self.req.local_file_path, allowZip64=True) as z: | |
return pkg_resources_distribution_for_wheel( | |
z, self.req.name, self.req.local_file_path | |
) | |
def prepare_distribution_metadata(self, finder, build_isolation): | |
# type: (PackageFinder, bool) -> None | |
pass |
"""Exceptions used throughout package""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
from itertools import chain, groupby, repeat | |
from pip._vendor.six import iteritems | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional | |
from pip._vendor.pkg_resources import Distribution | |
from pip._internal.req.req_install import InstallRequirement | |
class PipError(Exception): | |
"""Base pip exception""" | |
class ConfigurationError(PipError): | |
"""General exception in configuration""" | |
class InstallationError(PipError): | |
"""General exception during installation""" | |
class UninstallationError(PipError): | |
"""General exception during uninstallation""" | |
class NoneMetadataError(PipError): | |
""" | |
Raised when accessing "METADATA" or "PKG-INFO" metadata for a | |
pip._vendor.pkg_resources.Distribution object and | |
`dist.has_metadata('METADATA')` returns True but | |
`dist.get_metadata('METADATA')` returns None (and similarly for | |
"PKG-INFO"). | |
""" | |
def __init__(self, dist, metadata_name): | |
# type: (Distribution, str) -> None | |
""" | |
:param dist: A Distribution object. | |
:param metadata_name: The name of the metadata being accessed | |
(can be "METADATA" or "PKG-INFO"). | |
""" | |
self.dist = dist | |
self.metadata_name = metadata_name | |
def __str__(self): | |
# type: () -> str | |
# Use `dist` in the error message because its stringification | |
# includes more information, like the version and location. | |
return ( | |
'None {} metadata found for distribution: {}'.format( | |
self.metadata_name, self.dist, | |
) | |
) | |
class DistributionNotFound(InstallationError): | |
"""Raised when a distribution cannot be found to satisfy a requirement""" | |
class RequirementsFileParseError(InstallationError): | |
"""Raised when a general error occurs parsing a requirements file line.""" | |
class BestVersionAlreadyInstalled(PipError): | |
"""Raised when the most up-to-date version of a package is already | |
installed.""" | |
class BadCommand(PipError): | |
"""Raised when virtualenv or a command is not found""" | |
class CommandError(PipError): | |
"""Raised when there is an error in command-line arguments""" | |
class PreviousBuildDirError(PipError): | |
"""Raised when there's a previous conflicting build directory""" | |
class InvalidWheelFilename(InstallationError): | |
"""Invalid wheel filename.""" | |
class UnsupportedWheel(InstallationError): | |
"""Unsupported wheel.""" | |
class HashErrors(InstallationError): | |
"""Multiple HashError instances rolled into one for reporting""" | |
def __init__(self): | |
self.errors = [] | |
def append(self, error): | |
self.errors.append(error) | |
def __str__(self): | |
lines = [] | |
self.errors.sort(key=lambda e: e.order) | |
for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__): | |
lines.append(cls.head) | |
lines.extend(e.body() for e in errors_of_cls) | |
if lines: | |
return '\n'.join(lines) | |
def __nonzero__(self): | |
return bool(self.errors) | |
def __bool__(self): | |
return self.__nonzero__() | |
class HashError(InstallationError): | |
""" | |
A failure to verify a package against known-good hashes | |
:cvar order: An int sorting hash exception classes by difficulty of | |
recovery (lower being harder), so the user doesn't bother fretting | |
about unpinned packages when he has deeper issues, like VCS | |
dependencies, to deal with. Also keeps error reports in a | |
deterministic order. | |
:cvar head: A section heading for display above potentially many | |
exceptions of this kind | |
:ivar req: The InstallRequirement that triggered this error. This is | |
pasted on after the exception is instantiated, because it's not | |
typically available earlier. | |
""" | |
req = None # type: Optional[InstallRequirement] | |
head = '' | |
def body(self): | |
"""Return a summary of me for display under the heading. | |
This default implementation simply prints a description of the | |
triggering requirement. | |
:param req: The InstallRequirement that provoked this error, with | |
populate_link() having already been called | |
""" | |
return ' %s' % self._requirement_name() | |
def __str__(self): | |
return '%s\n%s' % (self.head, self.body()) | |
def _requirement_name(self): | |
"""Return a description of the requirement that triggered me. | |
This default implementation returns long description of the req, with | |
line numbers | |
""" | |
return str(self.req) if self.req else 'unknown package' | |
class VcsHashUnsupported(HashError): | |
"""A hash was provided for a version-control-system-based requirement, but | |
we don't have a method for hashing those.""" | |
order = 0 | |
head = ("Can't verify hashes for these requirements because we don't " | |
"have a way to hash version control repositories:") | |
class DirectoryUrlHashUnsupported(HashError): | |
"""A hash was provided for a version-control-system-based requirement, but | |
we don't have a method for hashing those.""" | |
order = 1 | |
head = ("Can't verify hashes for these file:// requirements because they " | |
"point to directories:") | |
class HashMissing(HashError): | |
"""A hash was needed for a requirement but is absent.""" | |
order = 2 | |
head = ('Hashes are required in --require-hashes mode, but they are ' | |
'missing from some requirements. Here is a list of those ' | |
'requirements along with the hashes their downloaded archives ' | |
'actually had. Add lines like these to your requirements files to ' | |
'prevent tampering. (If you did not enable --require-hashes ' | |
'manually, note that it turns on automatically when any package ' | |
'has a hash.)') | |
def __init__(self, gotten_hash): | |
""" | |
:param gotten_hash: The hash of the (possibly malicious) archive we | |
just downloaded | |
""" | |
self.gotten_hash = gotten_hash | |
def body(self): | |
# Dodge circular import. | |
from pip._internal.utils.hashes import FAVORITE_HASH | |
package = None | |
if self.req: | |
# In the case of URL-based requirements, display the original URL | |
# seen in the requirements file rather than the package name, | |
# so the output can be directly copied into the requirements file. | |
package = (self.req.original_link if self.req.original_link | |
# In case someone feeds something downright stupid | |
# to InstallRequirement's constructor. | |
else getattr(self.req, 'req', None)) | |
return ' %s --hash=%s:%s' % (package or 'unknown package', | |
FAVORITE_HASH, | |
self.gotten_hash) | |
class HashUnpinned(HashError): | |
"""A requirement had a hash specified but was not pinned to a specific | |
version.""" | |
order = 3 | |
head = ('In --require-hashes mode, all requirements must have their ' | |
'versions pinned with ==. These do not:') | |
class HashMismatch(HashError): | |
""" | |
Distribution file hash values don't match. | |
:ivar package_name: The name of the package that triggered the hash | |
mismatch. Feel free to write to this after the exception is raise to | |
improve its error message. | |
""" | |
order = 4 | |
head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS ' | |
'FILE. If you have updated the package versions, please update ' | |
'the hashes. Otherwise, examine the package contents carefully; ' | |
'someone may have tampered with them.') | |
def __init__(self, allowed, gots): | |
""" | |
:param allowed: A dict of algorithm names pointing to lists of allowed | |
hex digests | |
:param gots: A dict of algorithm names pointing to hashes we | |
actually got from the files under suspicion | |
""" | |
self.allowed = allowed | |
self.gots = gots | |
def body(self): | |
return ' %s:\n%s' % (self._requirement_name(), | |
self._hash_comparison()) | |
def _hash_comparison(self): | |
""" | |
Return a comparison of actual and expected hash values. | |
Example:: | |
Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde | |
or 123451234512345123451234512345123451234512345 | |
Got bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef | |
""" | |
def hash_then_or(hash_name): | |
# For now, all the decent hashes have 6-char names, so we can get | |
# away with hard-coding space literals. | |
return chain([hash_name], repeat(' or')) | |
lines = [] | |
for hash_name, expecteds in iteritems(self.allowed): | |
prefix = hash_then_or(hash_name) | |
lines.extend((' Expected %s %s' % (next(prefix), e)) | |
for e in expecteds) | |
lines.append(' Got %s\n' % | |
self.gots[hash_name].hexdigest()) | |
return '\n'.join(lines) | |
class UnsupportedPythonVersion(InstallationError): | |
"""Unsupported python version according to Requires-Python package | |
metadata.""" | |
class ConfigurationFileCouldNotBeLoaded(ConfigurationError): | |
"""When there are errors while loading a configuration file | |
""" | |
def __init__(self, reason="could not be loaded", fname=None, error=None): | |
super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) | |
self.reason = reason | |
self.fname = fname | |
self.error = error | |
def __str__(self): | |
if self.fname is not None: | |
message_part = " in {}.".format(self.fname) | |
else: | |
assert self.error is not None | |
message_part = ".\n{}\n".format(self.error.message) | |
return "Configuration file {}{}".format(self.reason, message_part) |
"""Index interaction code | |
""" |
""" | |
The main purpose of this module is to expose LinkCollector.collect_links(). | |
""" | |
import cgi | |
import itertools | |
import logging | |
import mimetypes | |
import os | |
from collections import OrderedDict | |
from pip._vendor import html5lib, requests | |
from pip._vendor.distlib.compat import unescape | |
from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._vendor.six.moves.urllib import request as urllib_request | |
from pip._internal.models.link import Link | |
from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS | |
from pip._internal.utils.misc import redact_auth_from_url | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url, url_to_path | |
from pip._internal.vcs import is_url, vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Callable, Iterable, List, MutableMapping, Optional, Sequence, Tuple, | |
Union, | |
) | |
import xml.etree.ElementTree | |
from pip._vendor.requests import Response | |
from pip._internal.models.search_scope import SearchScope | |
from pip._internal.network.session import PipSession | |
HTMLElement = xml.etree.ElementTree.Element | |
ResponseHeaders = MutableMapping[str, str] | |
logger = logging.getLogger(__name__) | |
def _match_vcs_scheme(url): | |
# type: (str) -> Optional[str] | |
"""Look for VCS schemes in the URL. | |
Returns the matched VCS scheme, or None if there's no match. | |
""" | |
for scheme in vcs.schemes: | |
if url.lower().startswith(scheme) and url[len(scheme)] in '+:': | |
return scheme | |
return None | |
def _is_url_like_archive(url): | |
# type: (str) -> bool | |
"""Return whether the URL looks like an archive. | |
""" | |
filename = Link(url).filename | |
for bad_ext in ARCHIVE_EXTENSIONS: | |
if filename.endswith(bad_ext): | |
return True | |
return False | |
class _NotHTML(Exception): | |
def __init__(self, content_type, request_desc): | |
# type: (str, str) -> None | |
super(_NotHTML, self).__init__(content_type, request_desc) | |
self.content_type = content_type | |
self.request_desc = request_desc | |
def _ensure_html_header(response): | |
# type: (Response) -> None | |
"""Check the Content-Type header to ensure the response contains HTML. | |
Raises `_NotHTML` if the content type is not text/html. | |
""" | |
content_type = response.headers.get("Content-Type", "") | |
if not content_type.lower().startswith("text/html"): | |
raise _NotHTML(content_type, response.request.method) | |
class _NotHTTP(Exception): | |
pass | |
def _ensure_html_response(url, session): | |
# type: (str, PipSession) -> None | |
"""Send a HEAD request to the URL, and ensure the response contains HTML. | |
Raises `_NotHTTP` if the URL is not available for a HEAD request, or | |
`_NotHTML` if the content type is not text/html. | |
""" | |
scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) | |
if scheme not in {'http', 'https'}: | |
raise _NotHTTP() | |
resp = session.head(url, allow_redirects=True) | |
resp.raise_for_status() | |
_ensure_html_header(resp) | |
def _get_html_response(url, session): | |
# type: (str, PipSession) -> Response | |
"""Access an HTML page with GET, and return the response. | |
This consists of three parts: | |
1. If the URL looks suspiciously like an archive, send a HEAD first to | |
check the Content-Type is HTML, to avoid downloading a large file. | |
Raise `_NotHTTP` if the content type cannot be determined, or | |
`_NotHTML` if it is not HTML. | |
2. Actually perform the request. Raise HTTP exceptions on network failures. | |
3. Check the Content-Type header to make sure we got HTML, and raise | |
`_NotHTML` otherwise. | |
""" | |
if _is_url_like_archive(url): | |
_ensure_html_response(url, session=session) | |
logger.debug('Getting page %s', redact_auth_from_url(url)) | |
resp = session.get( | |
url, | |
headers={ | |
"Accept": "text/html", | |
# We don't want to blindly returned cached data for | |
# /simple/, because authors generally expecting that | |
# twine upload && pip install will function, but if | |
# they've done a pip install in the last ~10 minutes | |
# it won't. Thus by setting this to zero we will not | |
# blindly use any cached data, however the benefit of | |
# using max-age=0 instead of no-cache, is that we will | |
# still support conditional requests, so we will still | |
# minimize traffic sent in cases where the page hasn't | |
# changed at all, we will just always incur the round | |
# trip for the conditional GET now instead of only | |
# once per 10 minutes. | |
# For more information, please see pypa/pip#5670. | |
"Cache-Control": "max-age=0", | |
}, | |
) | |
resp.raise_for_status() | |
# The check for archives above only works if the url ends with | |
# something that looks like an archive. However that is not a | |
# requirement of an url. Unless we issue a HEAD request on every | |
# url we cannot know ahead of time for sure if something is HTML | |
# or not. However we can check after we've downloaded it. | |
_ensure_html_header(resp) | |
return resp | |
def _get_encoding_from_headers(headers): | |
# type: (ResponseHeaders) -> Optional[str] | |
"""Determine if we have any encoding information in our headers. | |
""" | |
if headers and "Content-Type" in headers: | |
content_type, params = cgi.parse_header(headers["Content-Type"]) | |
if "charset" in params: | |
return params['charset'] | |
return None | |
def _determine_base_url(document, page_url): | |
# type: (HTMLElement, str) -> str | |
"""Determine the HTML document's base URL. | |
This looks for a ``<base>`` tag in the HTML document. If present, its href | |
attribute denotes the base URL of anchor tags in the document. If there is | |
no such tag (or if it does not have a valid href attribute), the HTML | |
file's URL is used as the base URL. | |
:param document: An HTML document representation. The current | |
implementation expects the result of ``html5lib.parse()``. | |
:param page_url: The URL of the HTML document. | |
""" | |
for base in document.findall(".//base"): | |
href = base.get("href") | |
if href is not None: | |
return href | |
return page_url | |
def _clean_link(url): | |
# type: (str) -> str | |
"""Makes sure a link is fully encoded. That is, if a ' ' shows up in | |
the link, it will be rewritten to %20 (while not over-quoting | |
% or other characters).""" | |
# Split the URL into parts according to the general structure | |
# `scheme://netloc/path;parameters?query#fragment`. Note that the | |
# `netloc` can be empty and the URI will then refer to a local | |
# filesystem path. | |
result = urllib_parse.urlparse(url) | |
# In both cases below we unquote prior to quoting to make sure | |
# nothing is double quoted. | |
if result.netloc == "": | |
# On Windows the path part might contain a drive letter which | |
# should not be quoted. On Linux where drive letters do not | |
# exist, the colon should be quoted. We rely on urllib.request | |
# to do the right thing here. | |
path = urllib_request.pathname2url( | |
urllib_request.url2pathname(result.path)) | |
else: | |
# In addition to the `/` character we protect `@` so that | |
# revision strings in VCS URLs are properly parsed. | |
path = urllib_parse.quote(urllib_parse.unquote(result.path), safe="/@") | |
return urllib_parse.urlunparse(result._replace(path=path)) | |
def _create_link_from_element( | |
anchor, # type: HTMLElement | |
page_url, # type: str | |
base_url, # type: str | |
): | |
# type: (...) -> Optional[Link] | |
""" | |
Convert an anchor element in a simple repository page to a Link. | |
""" | |
href = anchor.get("href") | |
if not href: | |
return None | |
url = _clean_link(urllib_parse.urljoin(base_url, href)) | |
pyrequire = anchor.get('data-requires-python') | |
pyrequire = unescape(pyrequire) if pyrequire else None | |
yanked_reason = anchor.get('data-yanked') | |
if yanked_reason: | |
# This is a unicode string in Python 2 (and 3). | |
yanked_reason = unescape(yanked_reason) | |
link = Link( | |
url, | |
comes_from=page_url, | |
requires_python=pyrequire, | |
yanked_reason=yanked_reason, | |
) | |
return link | |
def parse_links(page): | |
# type: (HTMLPage) -> Iterable[Link] | |
""" | |
Parse an HTML document, and yield its anchor elements as Link objects. | |
""" | |
document = html5lib.parse( | |
page.content, | |
transport_encoding=page.encoding, | |
namespaceHTMLElements=False, | |
) | |
url = page.url | |
base_url = _determine_base_url(document, url) | |
for anchor in document.findall(".//a"): | |
link = _create_link_from_element( | |
anchor, | |
page_url=url, | |
base_url=base_url, | |
) | |
if link is None: | |
continue | |
yield link | |
class HTMLPage(object): | |
"""Represents one page, along with its URL""" | |
def __init__( | |
self, | |
content, # type: bytes | |
encoding, # type: Optional[str] | |
url, # type: str | |
): | |
# type: (...) -> None | |
""" | |
:param encoding: the encoding to decode the given content. | |
:param url: the URL from which the HTML was downloaded. | |
""" | |
self.content = content | |
self.encoding = encoding | |
self.url = url | |
def __str__(self): | |
# type: () -> str | |
return redact_auth_from_url(self.url) | |
def _handle_get_page_fail( | |
link, # type: Link | |
reason, # type: Union[str, Exception] | |
meth=None # type: Optional[Callable[..., None]] | |
): | |
# type: (...) -> None | |
if meth is None: | |
meth = logger.debug | |
meth("Could not fetch URL %s: %s - skipping", link, reason) | |
def _make_html_page(response): | |
# type: (Response) -> HTMLPage | |
encoding = _get_encoding_from_headers(response.headers) | |
return HTMLPage(response.content, encoding=encoding, url=response.url) | |
def _get_html_page(link, session=None): | |
# type: (Link, Optional[PipSession]) -> Optional[HTMLPage] | |
if session is None: | |
raise TypeError( | |
"_get_html_page() missing 1 required keyword argument: 'session'" | |
) | |
url = link.url.split('#', 1)[0] | |
# Check for VCS schemes that do not support lookup as web pages. | |
vcs_scheme = _match_vcs_scheme(url) | |
if vcs_scheme: | |
logger.debug('Cannot look at %s URL %s', vcs_scheme, link) | |
return None | |
# Tack index.html onto file:// URLs that point to directories | |
scheme, _, path, _, _, _ = urllib_parse.urlparse(url) | |
if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): | |
# add trailing slash if not present so urljoin doesn't trim | |
# final segment | |
if not url.endswith('/'): | |
url += '/' | |
url = urllib_parse.urljoin(url, 'index.html') | |
logger.debug(' file: URL is directory, getting %s', url) | |
try: | |
resp = _get_html_response(url, session=session) | |
except _NotHTTP: | |
logger.debug( | |
'Skipping page %s because it looks like an archive, and cannot ' | |
'be checked by HEAD.', link, | |
) | |
except _NotHTML as exc: | |
logger.debug( | |
'Skipping page %s because the %s request got Content-Type: %s', | |
link, exc.request_desc, exc.content_type, | |
) | |
except HTTPError as exc: | |
_handle_get_page_fail(link, exc) | |
except RetryError as exc: | |
_handle_get_page_fail(link, exc) | |
except SSLError as exc: | |
reason = "There was a problem confirming the ssl certificate: " | |
reason += str(exc) | |
_handle_get_page_fail(link, reason, meth=logger.info) | |
except requests.ConnectionError as exc: | |
_handle_get_page_fail(link, "connection error: %s" % exc) | |
except requests.Timeout: | |
_handle_get_page_fail(link, "timed out") | |
else: | |
return _make_html_page(resp) | |
return None | |
def _remove_duplicate_links(links): | |
# type: (Iterable[Link]) -> List[Link] | |
""" | |
Return a list of links, with duplicates removed and ordering preserved. | |
""" | |
# We preserve the ordering when removing duplicates because we can. | |
return list(OrderedDict.fromkeys(links)) | |
def group_locations(locations, expand_dir=False): | |
# type: (Sequence[str], bool) -> Tuple[List[str], List[str]] | |
""" | |
Divide a list of locations into two groups: "files" (archives) and "urls." | |
:return: A pair of lists (files, urls). | |
""" | |
files = [] | |
urls = [] | |
# puts the url for the given file path into the appropriate list | |
def sort_path(path): | |
# type: (str) -> None | |
url = path_to_url(path) | |
if mimetypes.guess_type(url, strict=False)[0] == 'text/html': | |
urls.append(url) | |
else: | |
files.append(url) | |
for url in locations: | |
is_local_path = os.path.exists(url) | |
is_file_url = url.startswith('file:') | |
if is_local_path or is_file_url: | |
if is_local_path: | |
path = url | |
else: | |
path = url_to_path(url) | |
if os.path.isdir(path): | |
if expand_dir: | |
path = os.path.realpath(path) | |
for item in os.listdir(path): | |
sort_path(os.path.join(path, item)) | |
elif is_file_url: | |
urls.append(url) | |
else: | |
logger.warning( | |
"Path '{0}' is ignored: " | |
"it is a directory.".format(path), | |
) | |
elif os.path.isfile(path): | |
sort_path(path) | |
else: | |
logger.warning( | |
"Url '%s' is ignored: it is neither a file " | |
"nor a directory.", url, | |
) | |
elif is_url(url): | |
# Only add url with clear scheme | |
urls.append(url) | |
else: | |
logger.warning( | |
"Url '%s' is ignored. It is either a non-existing " | |
"path or lacks a specific scheme.", url, | |
) | |
return files, urls | |
class CollectedLinks(object): | |
""" | |
Encapsulates the return value of a call to LinkCollector.collect_links(). | |
The return value includes both URLs to project pages containing package | |
links, as well as individual package Link objects collected from other | |
sources. | |
This info is stored separately as: | |
(1) links from the configured file locations, | |
(2) links from the configured find_links, and | |
(3) urls to HTML project pages, as described by the PEP 503 simple | |
repository API. | |
""" | |
def __init__( | |
self, | |
files, # type: List[Link] | |
find_links, # type: List[Link] | |
project_urls, # type: List[Link] | |
): | |
# type: (...) -> None | |
""" | |
:param files: Links from file locations. | |
:param find_links: Links from find_links. | |
:param project_urls: URLs to HTML project pages, as described by | |
the PEP 503 simple repository API. | |
""" | |
self.files = files | |
self.find_links = find_links | |
self.project_urls = project_urls | |
class LinkCollector(object): | |
""" | |
Responsible for collecting Link objects from all configured locations, | |
making network requests as needed. | |
The class's main method is its collect_links() method. | |
""" | |
def __init__( | |
self, | |
session, # type: PipSession | |
search_scope, # type: SearchScope | |
): | |
# type: (...) -> None | |
self.search_scope = search_scope | |
self.session = session | |
@property | |
def find_links(self): | |
# type: () -> List[str] | |
return self.search_scope.find_links | |
def fetch_page(self, location): | |
# type: (Link) -> Optional[HTMLPage] | |
""" | |
Fetch an HTML page containing package links. | |
""" | |
return _get_html_page(location, session=self.session) | |
def collect_links(self, project_name): | |
# type: (str) -> CollectedLinks | |
"""Find all available links for the given project name. | |
:return: All the Link objects (unfiltered), as a CollectedLinks object. | |
""" | |
search_scope = self.search_scope | |
index_locations = search_scope.get_index_urls_locations(project_name) | |
index_file_loc, index_url_loc = group_locations(index_locations) | |
fl_file_loc, fl_url_loc = group_locations( | |
self.find_links, expand_dir=True, | |
) | |
file_links = [ | |
Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) | |
] | |
# We trust every directly linked archive in find_links | |
find_link_links = [Link(url, '-f') for url in self.find_links] | |
# We trust every url that the user has given us whether it was given | |
# via --index-url or --find-links. | |
# We want to filter out anything that does not have a secure origin. | |
url_locations = [ | |
link for link in itertools.chain( | |
(Link(url) for url in index_url_loc), | |
(Link(url) for url in fl_url_loc), | |
) | |
if self.session.is_secure_origin(link) | |
] | |
url_locations = _remove_duplicate_links(url_locations) | |
lines = [ | |
'{} location(s) to search for versions of {}:'.format( | |
len(url_locations), project_name, | |
), | |
] | |
for link in url_locations: | |
lines.append('* {}'.format(link)) | |
logger.debug('\n'.join(lines)) | |
return CollectedLinks( | |
files=file_links, | |
find_links=find_link_links, | |
project_urls=url_locations, | |
) |
"""Routines related to PyPI, indexes""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
import re | |
from pip._vendor.packaging import specifiers | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.packaging.version import parse as parse_version | |
from pip._internal.exceptions import ( | |
BestVersionAlreadyInstalled, | |
DistributionNotFound, | |
InvalidWheelFilename, | |
UnsupportedWheel, | |
) | |
from pip._internal.index.collector import parse_links | |
from pip._internal.models.candidate import InstallationCandidate | |
from pip._internal.models.format_control import FormatControl | |
from pip._internal.models.link import Link | |
from pip._internal.models.selection_prefs import SelectionPreferences | |
from pip._internal.models.target_python import TargetPython | |
from pip._internal.models.wheel import Wheel | |
from pip._internal.utils.filetypes import WHEEL_EXTENSION | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import build_netloc | |
from pip._internal.utils.packaging import check_requires_python | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS | |
from pip._internal.utils.urls import url_to_path | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union, | |
) | |
from pip._vendor.packaging.tags import Tag | |
from pip._vendor.packaging.version import _BaseVersion | |
from pip._internal.index.collector import LinkCollector | |
from pip._internal.models.search_scope import SearchScope | |
from pip._internal.req import InstallRequirement | |
from pip._internal.utils.hashes import Hashes | |
BuildTag = Union[Tuple[()], Tuple[int, str]] | |
CandidateSortingKey = ( | |
Tuple[int, int, int, _BaseVersion, BuildTag, Optional[int]] | |
) | |
__all__ = ['FormatControl', 'BestCandidateResult', 'PackageFinder'] | |
logger = logging.getLogger(__name__) | |
def _check_link_requires_python( | |
link, # type: Link | |
version_info, # type: Tuple[int, int, int] | |
ignore_requires_python=False, # type: bool | |
): | |
# type: (...) -> bool | |
""" | |
Return whether the given Python version is compatible with a link's | |
"Requires-Python" value. | |
:param version_info: A 3-tuple of ints representing the Python | |
major-minor-micro version to check. | |
:param ignore_requires_python: Whether to ignore the "Requires-Python" | |
value if the given Python version isn't compatible. | |
""" | |
try: | |
is_compatible = check_requires_python( | |
link.requires_python, version_info=version_info, | |
) | |
except specifiers.InvalidSpecifier: | |
logger.debug( | |
"Ignoring invalid Requires-Python (%r) for link: %s", | |
link.requires_python, link, | |
) | |
else: | |
if not is_compatible: | |
version = '.'.join(map(str, version_info)) | |
if not ignore_requires_python: | |
logger.debug( | |
'Link requires a different Python (%s not in: %r): %s', | |
version, link.requires_python, link, | |
) | |
return False | |
logger.debug( | |
'Ignoring failed Requires-Python check (%s not in: %r) ' | |
'for link: %s', | |
version, link.requires_python, link, | |
) | |
return True | |
class LinkEvaluator(object): | |
""" | |
Responsible for evaluating links for a particular project. | |
""" | |
_py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') | |
# Don't include an allow_yanked default value to make sure each call | |
# site considers whether yanked releases are allowed. This also causes | |
# that decision to be made explicit in the calling code, which helps | |
# people when reading the code. | |
def __init__( | |
self, | |
project_name, # type: str | |
canonical_name, # type: str | |
formats, # type: FrozenSet[str] | |
target_python, # type: TargetPython | |
allow_yanked, # type: bool | |
ignore_requires_python=None, # type: Optional[bool] | |
): | |
# type: (...) -> None | |
""" | |
:param project_name: The user supplied package name. | |
:param canonical_name: The canonical package name. | |
:param formats: The formats allowed for this package. Should be a set | |
with 'binary' or 'source' or both in it. | |
:param target_python: The target Python interpreter to use when | |
evaluating link compatibility. This is used, for example, to | |
check wheel compatibility, as well as when checking the Python | |
version, e.g. the Python version embedded in a link filename | |
(or egg fragment) and against an HTML link's optional PEP 503 | |
"data-requires-python" attribute. | |
:param allow_yanked: Whether files marked as yanked (in the sense | |
of PEP 592) are permitted to be candidates for install. | |
:param ignore_requires_python: Whether to ignore incompatible | |
PEP 503 "data-requires-python" values in HTML links. Defaults | |
to False. | |
""" | |
if ignore_requires_python is None: | |
ignore_requires_python = False | |
self._allow_yanked = allow_yanked | |
self._canonical_name = canonical_name | |
self._ignore_requires_python = ignore_requires_python | |
self._formats = formats | |
self._target_python = target_python | |
self.project_name = project_name | |
def evaluate_link(self, link): | |
# type: (Link) -> Tuple[bool, Optional[Text]] | |
""" | |
Determine whether a link is a candidate for installation. | |
:return: A tuple (is_candidate, result), where `result` is (1) a | |
version string if `is_candidate` is True, and (2) if | |
`is_candidate` is False, an optional string to log the reason | |
the link fails to qualify. | |
""" | |
version = None | |
if link.is_yanked and not self._allow_yanked: | |
reason = link.yanked_reason or '<none given>' | |
# Mark this as a unicode string to prevent "UnicodeEncodeError: | |
# 'ascii' codec can't encode character" in Python 2 when | |
# the reason contains non-ascii characters. | |
return (False, u'yanked for reason: {}'.format(reason)) | |
if link.egg_fragment: | |
egg_info = link.egg_fragment | |
ext = link.ext | |
else: | |
egg_info, ext = link.splitext() | |
if not ext: | |
return (False, 'not a file') | |
if ext not in SUPPORTED_EXTENSIONS: | |
return (False, 'unsupported archive format: %s' % ext) | |
if "binary" not in self._formats and ext == WHEEL_EXTENSION: | |
reason = 'No binaries permitted for %s' % self.project_name | |
return (False, reason) | |
if "macosx10" in link.path and ext == '.zip': | |
return (False, 'macosx10 one') | |
if ext == WHEEL_EXTENSION: | |
try: | |
wheel = Wheel(link.filename) | |
except InvalidWheelFilename: | |
return (False, 'invalid wheel filename') | |
if canonicalize_name(wheel.name) != self._canonical_name: | |
reason = 'wrong project name (not %s)' % self.project_name | |
return (False, reason) | |
supported_tags = self._target_python.get_tags() | |
if not wheel.supported(supported_tags): | |
# Include the wheel's tags in the reason string to | |
# simplify troubleshooting compatibility issues. | |
file_tags = wheel.get_formatted_file_tags() | |
reason = ( | |
"none of the wheel's tags match: {}".format( | |
', '.join(file_tags) | |
) | |
) | |
return (False, reason) | |
version = wheel.version | |
# This should be up by the self.ok_binary check, but see issue 2700. | |
if "source" not in self._formats and ext != WHEEL_EXTENSION: | |
return (False, 'No sources permitted for %s' % self.project_name) | |
if not version: | |
version = _extract_version_from_fragment( | |
egg_info, self._canonical_name, | |
) | |
if not version: | |
return ( | |
False, 'Missing project version for %s' % self.project_name, | |
) | |
match = self._py_version_re.search(version) | |
if match: | |
version = version[:match.start()] | |
py_version = match.group(1) | |
if py_version != self._target_python.py_version: | |
return (False, 'Python version is incorrect') | |
supports_python = _check_link_requires_python( | |
link, version_info=self._target_python.py_version_info, | |
ignore_requires_python=self._ignore_requires_python, | |
) | |
if not supports_python: | |
# Return None for the reason text to suppress calling | |
# _log_skipped_link(). | |
return (False, None) | |
logger.debug('Found link %s, version: %s', link, version) | |
return (True, version) | |
def filter_unallowed_hashes( | |
candidates, # type: List[InstallationCandidate] | |
hashes, # type: Hashes | |
project_name, # type: str | |
): | |
# type: (...) -> List[InstallationCandidate] | |
""" | |
Filter out candidates whose hashes aren't allowed, and return a new | |
list of candidates. | |
If at least one candidate has an allowed hash, then all candidates with | |
either an allowed hash or no hash specified are returned. Otherwise, | |
the given candidates are returned. | |
Including the candidates with no hash specified when there is a match | |
allows a warning to be logged if there is a more preferred candidate | |
with no hash specified. Returning all candidates in the case of no | |
matches lets pip report the hash of the candidate that would otherwise | |
have been installed (e.g. permitting the user to more easily update | |
their requirements file with the desired hash). | |
""" | |
if not hashes: | |
logger.debug( | |
'Given no hashes to check %s links for project %r: ' | |
'discarding no candidates', | |
len(candidates), | |
project_name, | |
) | |
# Make sure we're not returning back the given value. | |
return list(candidates) | |
matches_or_no_digest = [] | |
# Collect the non-matches for logging purposes. | |
non_matches = [] | |
match_count = 0 | |
for candidate in candidates: | |
link = candidate.link | |
if not link.has_hash: | |
pass | |
elif link.is_hash_allowed(hashes=hashes): | |
match_count += 1 | |
else: | |
non_matches.append(candidate) | |
continue | |
matches_or_no_digest.append(candidate) | |
if match_count: | |
filtered = matches_or_no_digest | |
else: | |
# Make sure we're not returning back the given value. | |
filtered = list(candidates) | |
if len(filtered) == len(candidates): | |
discard_message = 'discarding no candidates' | |
else: | |
discard_message = 'discarding {} non-matches:\n {}'.format( | |
len(non_matches), | |
'\n '.join(str(candidate.link) for candidate in non_matches) | |
) | |
logger.debug( | |
'Checked %s links for project %r against %s hashes ' | |
'(%s matches, %s no digest): %s', | |
len(candidates), | |
project_name, | |
hashes.digest_count, | |
match_count, | |
len(matches_or_no_digest) - match_count, | |
discard_message | |
) | |
return filtered | |
class CandidatePreferences(object): | |
""" | |
Encapsulates some of the preferences for filtering and sorting | |
InstallationCandidate objects. | |
""" | |
def __init__( | |
self, | |
prefer_binary=False, # type: bool | |
allow_all_prereleases=False, # type: bool | |
): | |
# type: (...) -> None | |
""" | |
:param allow_all_prereleases: Whether to allow all pre-releases. | |
""" | |
self.allow_all_prereleases = allow_all_prereleases | |
self.prefer_binary = prefer_binary | |
class BestCandidateResult(object): | |
"""A collection of candidates, returned by `PackageFinder.find_best_candidate`. | |
This class is only intended to be instantiated by CandidateEvaluator's | |
`compute_best_candidate()` method. | |
""" | |
def __init__( | |
self, | |
candidates, # type: List[InstallationCandidate] | |
applicable_candidates, # type: List[InstallationCandidate] | |
best_candidate, # type: Optional[InstallationCandidate] | |
): | |
# type: (...) -> None | |
""" | |
:param candidates: A sequence of all available candidates found. | |
:param applicable_candidates: The applicable candidates. | |
:param best_candidate: The most preferred candidate found, or None | |
if no applicable candidates were found. | |
""" | |
assert set(applicable_candidates) <= set(candidates) | |
if best_candidate is None: | |
assert not applicable_candidates | |
else: | |
assert best_candidate in applicable_candidates | |
self._applicable_candidates = applicable_candidates | |
self._candidates = candidates | |
self.best_candidate = best_candidate | |
def iter_all(self): | |
# type: () -> Iterable[InstallationCandidate] | |
"""Iterate through all candidates. | |
""" | |
return iter(self._candidates) | |
def iter_applicable(self): | |
# type: () -> Iterable[InstallationCandidate] | |
"""Iterate through the applicable candidates. | |
""" | |
return iter(self._applicable_candidates) | |
class CandidateEvaluator(object): | |
""" | |
Responsible for filtering and sorting candidates for installation based | |
on what tags are valid. | |
""" | |
@classmethod | |
def create( | |
cls, | |
project_name, # type: str | |
target_python=None, # type: Optional[TargetPython] | |
prefer_binary=False, # type: bool | |
allow_all_prereleases=False, # type: bool | |
specifier=None, # type: Optional[specifiers.BaseSpecifier] | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> CandidateEvaluator | |
"""Create a CandidateEvaluator object. | |
:param target_python: The target Python interpreter to use when | |
checking compatibility. If None (the default), a TargetPython | |
object will be constructed from the running Python. | |
:param specifier: An optional object implementing `filter` | |
(e.g. `packaging.specifiers.SpecifierSet`) to filter applicable | |
versions. | |
:param hashes: An optional collection of allowed hashes. | |
""" | |
if target_python is None: | |
target_python = TargetPython() | |
if specifier is None: | |
specifier = specifiers.SpecifierSet() | |
supported_tags = target_python.get_tags() | |
return cls( | |
project_name=project_name, | |
supported_tags=supported_tags, | |
specifier=specifier, | |
prefer_binary=prefer_binary, | |
allow_all_prereleases=allow_all_prereleases, | |
hashes=hashes, | |
) | |
def __init__( | |
self, | |
project_name, # type: str | |
supported_tags, # type: List[Tag] | |
specifier, # type: specifiers.BaseSpecifier | |
prefer_binary=False, # type: bool | |
allow_all_prereleases=False, # type: bool | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> None | |
""" | |
:param supported_tags: The PEP 425 tags supported by the target | |
Python in order of preference (most preferred first). | |
""" | |
self._allow_all_prereleases = allow_all_prereleases | |
self._hashes = hashes | |
self._prefer_binary = prefer_binary | |
self._project_name = project_name | |
self._specifier = specifier | |
self._supported_tags = supported_tags | |
def get_applicable_candidates( | |
self, | |
candidates, # type: List[InstallationCandidate] | |
): | |
# type: (...) -> List[InstallationCandidate] | |
""" | |
Return the applicable candidates from a list of candidates. | |
""" | |
# Using None infers from the specifier instead. | |
allow_prereleases = self._allow_all_prereleases or None | |
specifier = self._specifier | |
versions = { | |
str(v) for v in specifier.filter( | |
# We turn the version object into a str here because otherwise | |
# when we're debundled but setuptools isn't, Python will see | |
# packaging.version.Version and | |
# pkg_resources._vendor.packaging.version.Version as different | |
# types. This way we'll use a str as a common data interchange | |
# format. If we stop using the pkg_resources provided specifier | |
# and start using our own, we can drop the cast to str(). | |
(str(c.version) for c in candidates), | |
prereleases=allow_prereleases, | |
) | |
} | |
# Again, converting version to str to deal with debundling. | |
applicable_candidates = [ | |
c for c in candidates if str(c.version) in versions | |
] | |
filtered_applicable_candidates = filter_unallowed_hashes( | |
candidates=applicable_candidates, | |
hashes=self._hashes, | |
project_name=self._project_name, | |
) | |
return sorted(filtered_applicable_candidates, key=self._sort_key) | |
def _sort_key(self, candidate): | |
# type: (InstallationCandidate) -> CandidateSortingKey | |
""" | |
Function to pass as the `key` argument to a call to sorted() to sort | |
InstallationCandidates by preference. | |
Returns a tuple such that tuples sorting as greater using Python's | |
default comparison operator are more preferred. | |
The preference is as follows: | |
First and foremost, candidates with allowed (matching) hashes are | |
always preferred over candidates without matching hashes. This is | |
because e.g. if the only candidate with an allowed hash is yanked, | |
we still want to use that candidate. | |
Second, excepting hash considerations, candidates that have been | |
yanked (in the sense of PEP 592) are always less preferred than | |
candidates that haven't been yanked. Then: | |
If not finding wheels, they are sorted by version only. | |
If finding wheels, then the sort order is by version, then: | |
1. existing installs | |
2. wheels ordered via Wheel.support_index_min(self._supported_tags) | |
3. source archives | |
If prefer_binary was set, then all wheels are sorted above sources. | |
Note: it was considered to embed this logic into the Link | |
comparison operators, but then different sdist links | |
with the same version, would have to be considered equal | |
""" | |
valid_tags = self._supported_tags | |
support_num = len(valid_tags) | |
build_tag = () # type: BuildTag | |
binary_preference = 0 | |
link = candidate.link | |
if link.is_wheel: | |
# can raise InvalidWheelFilename | |
wheel = Wheel(link.filename) | |
if not wheel.supported(valid_tags): | |
raise UnsupportedWheel( | |
"%s is not a supported wheel for this platform. It " | |
"can't be sorted." % wheel.filename | |
) | |
if self._prefer_binary: | |
binary_preference = 1 | |
pri = -(wheel.support_index_min(valid_tags)) | |
if wheel.build_tag is not None: | |
match = re.match(r'^(\d+)(.*)$', wheel.build_tag) | |
build_tag_groups = match.groups() | |
build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) | |
else: # sdist | |
pri = -(support_num) | |
has_allowed_hash = int(link.is_hash_allowed(self._hashes)) | |
yank_value = -1 * int(link.is_yanked) # -1 for yanked. | |
return ( | |
has_allowed_hash, yank_value, binary_preference, candidate.version, | |
build_tag, pri, | |
) | |
def sort_best_candidate( | |
self, | |
candidates, # type: List[InstallationCandidate] | |
): | |
# type: (...) -> Optional[InstallationCandidate] | |
""" | |
Return the best candidate per the instance's sort order, or None if | |
no candidate is acceptable. | |
""" | |
if not candidates: | |
return None | |
best_candidate = max(candidates, key=self._sort_key) | |
# Log a warning per PEP 592 if necessary before returning. | |
link = best_candidate.link | |
if link.is_yanked: | |
reason = link.yanked_reason or '<none given>' | |
msg = ( | |
# Mark this as a unicode string to prevent | |
# "UnicodeEncodeError: 'ascii' codec can't encode character" | |
# in Python 2 when the reason contains non-ascii characters. | |
u'The candidate selected for download or install is a ' | |
'yanked version: {candidate}\n' | |
'Reason for being yanked: {reason}' | |
).format(candidate=best_candidate, reason=reason) | |
logger.warning(msg) | |
return best_candidate | |
def compute_best_candidate( | |
self, | |
candidates, # type: List[InstallationCandidate] | |
): | |
# type: (...) -> BestCandidateResult | |
""" | |
Compute and return a `BestCandidateResult` instance. | |
""" | |
applicable_candidates = self.get_applicable_candidates(candidates) | |
best_candidate = self.sort_best_candidate(applicable_candidates) | |
return BestCandidateResult( | |
candidates, | |
applicable_candidates=applicable_candidates, | |
best_candidate=best_candidate, | |
) | |
class PackageFinder(object): | |
"""This finds packages. | |
This is meant to match easy_install's technique for looking for | |
packages, by reading pages and looking for appropriate links. | |
""" | |
def __init__( | |
self, | |
link_collector, # type: LinkCollector | |
target_python, # type: TargetPython | |
allow_yanked, # type: bool | |
format_control=None, # type: Optional[FormatControl] | |
candidate_prefs=None, # type: CandidatePreferences | |
ignore_requires_python=None, # type: Optional[bool] | |
): | |
# type: (...) -> None | |
""" | |
This constructor is primarily meant to be used by the create() class | |
method and from tests. | |
:param format_control: A FormatControl object, used to control | |
the selection of source packages / binary packages when consulting | |
the index and links. | |
:param candidate_prefs: Options to use when creating a | |
CandidateEvaluator object. | |
""" | |
if candidate_prefs is None: | |
candidate_prefs = CandidatePreferences() | |
format_control = format_control or FormatControl(set(), set()) | |
self._allow_yanked = allow_yanked | |
self._candidate_prefs = candidate_prefs | |
self._ignore_requires_python = ignore_requires_python | |
self._link_collector = link_collector | |
self._target_python = target_python | |
self.format_control = format_control | |
# These are boring links that have already been logged somehow. | |
self._logged_links = set() # type: Set[Link] | |
# Don't include an allow_yanked default value to make sure each call | |
# site considers whether yanked releases are allowed. This also causes | |
# that decision to be made explicit in the calling code, which helps | |
# people when reading the code. | |
@classmethod | |
def create( | |
cls, | |
link_collector, # type: LinkCollector | |
selection_prefs, # type: SelectionPreferences | |
target_python=None, # type: Optional[TargetPython] | |
): | |
# type: (...) -> PackageFinder | |
"""Create a PackageFinder. | |
:param selection_prefs: The candidate selection preferences, as a | |
SelectionPreferences object. | |
:param target_python: The target Python interpreter to use when | |
checking compatibility. If None (the default), a TargetPython | |
object will be constructed from the running Python. | |
""" | |
if target_python is None: | |
target_python = TargetPython() | |
candidate_prefs = CandidatePreferences( | |
prefer_binary=selection_prefs.prefer_binary, | |
allow_all_prereleases=selection_prefs.allow_all_prereleases, | |
) | |
return cls( | |
candidate_prefs=candidate_prefs, | |
link_collector=link_collector, | |
target_python=target_python, | |
allow_yanked=selection_prefs.allow_yanked, | |
format_control=selection_prefs.format_control, | |
ignore_requires_python=selection_prefs.ignore_requires_python, | |
) | |
@property | |
def search_scope(self): | |
# type: () -> SearchScope | |
return self._link_collector.search_scope | |
@search_scope.setter | |
def search_scope(self, search_scope): | |
# type: (SearchScope) -> None | |
self._link_collector.search_scope = search_scope | |
@property | |
def find_links(self): | |
# type: () -> List[str] | |
return self._link_collector.find_links | |
@property | |
def index_urls(self): | |
# type: () -> List[str] | |
return self.search_scope.index_urls | |
@property | |
def trusted_hosts(self): | |
# type: () -> Iterable[str] | |
for host_port in self._link_collector.session.pip_trusted_origins: | |
yield build_netloc(*host_port) | |
@property | |
def allow_all_prereleases(self): | |
# type: () -> bool | |
return self._candidate_prefs.allow_all_prereleases | |
def set_allow_all_prereleases(self): | |
# type: () -> None | |
self._candidate_prefs.allow_all_prereleases = True | |
def make_link_evaluator(self, project_name): | |
# type: (str) -> LinkEvaluator | |
canonical_name = canonicalize_name(project_name) | |
formats = self.format_control.get_allowed_formats(canonical_name) | |
return LinkEvaluator( | |
project_name=project_name, | |
canonical_name=canonical_name, | |
formats=formats, | |
target_python=self._target_python, | |
allow_yanked=self._allow_yanked, | |
ignore_requires_python=self._ignore_requires_python, | |
) | |
def _sort_links(self, links): | |
# type: (Iterable[Link]) -> List[Link] | |
""" | |
Returns elements of links in order, non-egg links first, egg links | |
second, while eliminating duplicates | |
""" | |
eggs, no_eggs = [], [] | |
seen = set() # type: Set[Link] | |
for link in links: | |
if link not in seen: | |
seen.add(link) | |
if link.egg_fragment: | |
eggs.append(link) | |
else: | |
no_eggs.append(link) | |
return no_eggs + eggs | |
def _log_skipped_link(self, link, reason): | |
# type: (Link, Text) -> None | |
if link not in self._logged_links: | |
# Mark this as a unicode string to prevent "UnicodeEncodeError: | |
# 'ascii' codec can't encode character" in Python 2 when | |
# the reason contains non-ascii characters. | |
# Also, put the link at the end so the reason is more visible | |
# and because the link string is usually very long. | |
logger.debug(u'Skipping link: %s: %s', reason, link) | |
self._logged_links.add(link) | |
def get_install_candidate(self, link_evaluator, link): | |
# type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] | |
""" | |
If the link is a candidate for install, convert it to an | |
InstallationCandidate and return it. Otherwise, return None. | |
""" | |
is_candidate, result = link_evaluator.evaluate_link(link) | |
if not is_candidate: | |
if result: | |
self._log_skipped_link(link, reason=result) | |
return None | |
return InstallationCandidate( | |
name=link_evaluator.project_name, | |
link=link, | |
# Convert the Text result to str since InstallationCandidate | |
# accepts str. | |
version=str(result), | |
) | |
def evaluate_links(self, link_evaluator, links): | |
# type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] | |
""" | |
Convert links that are candidates to InstallationCandidate objects. | |
""" | |
candidates = [] | |
for link in self._sort_links(links): | |
candidate = self.get_install_candidate(link_evaluator, link) | |
if candidate is not None: | |
candidates.append(candidate) | |
return candidates | |
def process_project_url(self, project_url, link_evaluator): | |
# type: (Link, LinkEvaluator) -> List[InstallationCandidate] | |
logger.debug( | |
'Fetching project page and analyzing links: %s', project_url, | |
) | |
html_page = self._link_collector.fetch_page(project_url) | |
if html_page is None: | |
return [] | |
page_links = list(parse_links(html_page)) | |
with indent_log(): | |
package_links = self.evaluate_links( | |
link_evaluator, | |
links=page_links, | |
) | |
return package_links | |
def find_all_candidates(self, project_name): | |
# type: (str) -> List[InstallationCandidate] | |
"""Find all available InstallationCandidate for project_name | |
This checks index_urls and find_links. | |
All versions found are returned as an InstallationCandidate list. | |
See LinkEvaluator.evaluate_link() for details on which files | |
are accepted. | |
""" | |
collected_links = self._link_collector.collect_links(project_name) | |
link_evaluator = self.make_link_evaluator(project_name) | |
find_links_versions = self.evaluate_links( | |
link_evaluator, | |
links=collected_links.find_links, | |
) | |
page_versions = [] | |
for project_url in collected_links.project_urls: | |
package_links = self.process_project_url( | |
project_url, link_evaluator=link_evaluator, | |
) | |
page_versions.extend(package_links) | |
file_versions = self.evaluate_links( | |
link_evaluator, | |
links=collected_links.files, | |
) | |
if file_versions: | |
file_versions.sort(reverse=True) | |
logger.debug( | |
'Local files found: %s', | |
', '.join([ | |
url_to_path(candidate.link.url) | |
for candidate in file_versions | |
]) | |
) | |
# This is an intentional priority ordering | |
return file_versions + find_links_versions + page_versions | |
def make_candidate_evaluator( | |
self, | |
project_name, # type: str | |
specifier=None, # type: Optional[specifiers.BaseSpecifier] | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> CandidateEvaluator | |
"""Create a CandidateEvaluator object to use. | |
""" | |
candidate_prefs = self._candidate_prefs | |
return CandidateEvaluator.create( | |
project_name=project_name, | |
target_python=self._target_python, | |
prefer_binary=candidate_prefs.prefer_binary, | |
allow_all_prereleases=candidate_prefs.allow_all_prereleases, | |
specifier=specifier, | |
hashes=hashes, | |
) | |
def find_best_candidate( | |
self, | |
project_name, # type: str | |
specifier=None, # type: Optional[specifiers.BaseSpecifier] | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> BestCandidateResult | |
"""Find matches for the given project and specifier. | |
:param specifier: An optional object implementing `filter` | |
(e.g. `packaging.specifiers.SpecifierSet`) to filter applicable | |
versions. | |
:return: A `BestCandidateResult` instance. | |
""" | |
candidates = self.find_all_candidates(project_name) | |
candidate_evaluator = self.make_candidate_evaluator( | |
project_name=project_name, | |
specifier=specifier, | |
hashes=hashes, | |
) | |
return candidate_evaluator.compute_best_candidate(candidates) | |
def find_requirement(self, req, upgrade): | |
# type: (InstallRequirement, bool) -> Optional[Link] | |
"""Try to find a Link matching req | |
Expects req, an InstallRequirement and upgrade, a boolean | |
Returns a Link if found, | |
Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise | |
""" | |
hashes = req.hashes(trust_internet=False) | |
best_candidate_result = self.find_best_candidate( | |
req.name, specifier=req.specifier, hashes=hashes, | |
) | |
best_candidate = best_candidate_result.best_candidate | |
installed_version = None # type: Optional[_BaseVersion] | |
if req.satisfied_by is not None: | |
installed_version = parse_version(req.satisfied_by.version) | |
def _format_versions(cand_iter): | |
# type: (Iterable[InstallationCandidate]) -> str | |
# This repeated parse_version and str() conversion is needed to | |
# handle different vendoring sources from pip and pkg_resources. | |
# If we stop using the pkg_resources provided specifier and start | |
# using our own, we can drop the cast to str(). | |
return ", ".join(sorted( | |
{str(c.version) for c in cand_iter}, | |
key=parse_version, | |
)) or "none" | |
if installed_version is None and best_candidate is None: | |
logger.critical( | |
'Could not find a version that satisfies the requirement %s ' | |
'(from versions: %s)', | |
req, | |
_format_versions(best_candidate_result.iter_all()), | |
) | |
raise DistributionNotFound( | |
'No matching distribution found for %s' % req | |
) | |
best_installed = False | |
if installed_version and ( | |
best_candidate is None or | |
best_candidate.version <= installed_version): | |
best_installed = True | |
if not upgrade and installed_version is not None: | |
if best_installed: | |
logger.debug( | |
'Existing installed version (%s) is most up-to-date and ' | |
'satisfies requirement', | |
installed_version, | |
) | |
else: | |
logger.debug( | |
'Existing installed version (%s) satisfies requirement ' | |
'(most up-to-date version is %s)', | |
installed_version, | |
best_candidate.version, | |
) | |
return None | |
if best_installed: | |
# We have an existing version, and its the best version | |
logger.debug( | |
'Installed version (%s) is most up-to-date (past versions: ' | |
'%s)', | |
installed_version, | |
_format_versions(best_candidate_result.iter_applicable()), | |
) | |
raise BestVersionAlreadyInstalled | |
logger.debug( | |
'Using version %s (newest of versions: %s)', | |
best_candidate.version, | |
_format_versions(best_candidate_result.iter_applicable()), | |
) | |
return best_candidate.link | |
def _find_name_version_sep(fragment, canonical_name): | |
# type: (str, str) -> int | |
"""Find the separator's index based on the package's canonical name. | |
:param fragment: A <package>+<version> filename "fragment" (stem) or | |
egg fragment. | |
:param canonical_name: The package's canonical name. | |
This function is needed since the canonicalized name does not necessarily | |
have the same length as the egg info's name part. An example:: | |
>>> fragment = 'foo__bar-1.0' | |
>>> canonical_name = 'foo-bar' | |
>>> _find_name_version_sep(fragment, canonical_name) | |
8 | |
""" | |
# Project name and version must be separated by one single dash. Find all | |
# occurrences of dashes; if the string in front of it matches the canonical | |
# name, this is the one separating the name and version parts. | |
for i, c in enumerate(fragment): | |
if c != "-": | |
continue | |
if canonicalize_name(fragment[:i]) == canonical_name: | |
return i | |
raise ValueError("{} does not match {}".format(fragment, canonical_name)) | |
def _extract_version_from_fragment(fragment, canonical_name): | |
# type: (str, str) -> Optional[str] | |
"""Parse the version string from a <package>+<version> filename | |
"fragment" (stem) or egg fragment. | |
:param fragment: The string to parse. E.g. foo-2.1 | |
:param canonical_name: The canonicalized name of the package this | |
belongs to. | |
""" | |
try: | |
version_start = _find_name_version_sep(fragment, canonical_name) + 1 | |
except ValueError: | |
return None | |
version = fragment[version_start:] | |
if not version: | |
return None | |
return version |
"""Dependency Resolution | |
The dependency resolution in pip is performed as follows: | |
for top-level requirements: | |
a. only one spec allowed per project, regardless of conflicts or not. | |
otherwise a "double requirement" exception is raised | |
b. they override sub-dependency requirements. | |
for sub-dependencies | |
a. "first found, wins" (where the order is breadth first) | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
import logging | |
import sys | |
from collections import defaultdict | |
from itertools import chain | |
from pip._vendor.packaging import specifiers | |
from pip._internal.exceptions import ( | |
BestVersionAlreadyInstalled, | |
DistributionNotFound, | |
HashError, | |
HashErrors, | |
UnsupportedPythonVersion, | |
) | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import dist_in_usersite, normalize_version_info | |
from pip._internal.utils.packaging import ( | |
check_requires_python, | |
get_requires_python, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Callable, DefaultDict, List, Optional, Set, Tuple | |
from pip._vendor import pkg_resources | |
from pip._internal.distributions import AbstractDistribution | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.operations.prepare import RequirementPreparer | |
from pip._internal.req.req_install import InstallRequirement | |
from pip._internal.req.req_set import RequirementSet | |
InstallRequirementProvider = Callable[ | |
[str, InstallRequirement], InstallRequirement | |
] | |
DiscoveredDependencies = DefaultDict[str, List[InstallRequirement]] | |
logger = logging.getLogger(__name__) | |
def _check_dist_requires_python( | |
dist, # type: pkg_resources.Distribution | |
version_info, # type: Tuple[int, int, int] | |
ignore_requires_python=False, # type: bool | |
): | |
# type: (...) -> None | |
""" | |
Check whether the given Python version is compatible with a distribution's | |
"Requires-Python" value. | |
:param version_info: A 3-tuple of ints representing the Python | |
major-minor-micro version to check. | |
:param ignore_requires_python: Whether to ignore the "Requires-Python" | |
value if the given Python version isn't compatible. | |
:raises UnsupportedPythonVersion: When the given Python version isn't | |
compatible. | |
""" | |
requires_python = get_requires_python(dist) | |
try: | |
is_compatible = check_requires_python( | |
requires_python, version_info=version_info, | |
) | |
except specifiers.InvalidSpecifier as exc: | |
logger.warning( | |
"Package %r has an invalid Requires-Python: %s", | |
dist.project_name, exc, | |
) | |
return | |
if is_compatible: | |
return | |
version = '.'.join(map(str, version_info)) | |
if ignore_requires_python: | |
logger.debug( | |
'Ignoring failed Requires-Python check for package %r: ' | |
'%s not in %r', | |
dist.project_name, version, requires_python, | |
) | |
return | |
raise UnsupportedPythonVersion( | |
'Package {!r} requires a different Python: {} not in {!r}'.format( | |
dist.project_name, version, requires_python, | |
)) | |
class Resolver(object): | |
"""Resolves which packages need to be installed/uninstalled to perform \ | |
the requested operation without breaking the requirements of any package. | |
""" | |
_allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} | |
def __init__( | |
self, | |
preparer, # type: RequirementPreparer | |
finder, # type: PackageFinder | |
make_install_req, # type: InstallRequirementProvider | |
use_user_site, # type: bool | |
ignore_dependencies, # type: bool | |
ignore_installed, # type: bool | |
ignore_requires_python, # type: bool | |
force_reinstall, # type: bool | |
upgrade_strategy, # type: str | |
py_version_info=None, # type: Optional[Tuple[int, ...]] | |
): | |
# type: (...) -> None | |
super(Resolver, self).__init__() | |
assert upgrade_strategy in self._allowed_strategies | |
if py_version_info is None: | |
py_version_info = sys.version_info[:3] | |
else: | |
py_version_info = normalize_version_info(py_version_info) | |
self._py_version_info = py_version_info | |
self.preparer = preparer | |
self.finder = finder | |
self.upgrade_strategy = upgrade_strategy | |
self.force_reinstall = force_reinstall | |
self.ignore_dependencies = ignore_dependencies | |
self.ignore_installed = ignore_installed | |
self.ignore_requires_python = ignore_requires_python | |
self.use_user_site = use_user_site | |
self._make_install_req = make_install_req | |
self._discovered_dependencies = \ | |
defaultdict(list) # type: DiscoveredDependencies | |
def resolve(self, requirement_set): | |
# type: (RequirementSet) -> None | |
"""Resolve what operations need to be done | |
As a side-effect of this method, the packages (and their dependencies) | |
are downloaded, unpacked and prepared for installation. This | |
preparation is done by ``pip.operations.prepare``. | |
Once PyPI has static dependency metadata available, it would be | |
possible to move the preparation to become a step separated from | |
dependency resolution. | |
""" | |
# If any top-level requirement has a hash specified, enter | |
# hash-checking mode, which requires hashes from all. | |
root_reqs = ( | |
requirement_set.unnamed_requirements + | |
list(requirement_set.requirements.values()) | |
) | |
# Actually prepare the files, and collect any exceptions. Most hash | |
# exceptions cannot be checked ahead of time, because | |
# req.populate_link() needs to be called before we can make decisions | |
# based on link type. | |
discovered_reqs = [] # type: List[InstallRequirement] | |
hash_errors = HashErrors() | |
for req in chain(root_reqs, discovered_reqs): | |
try: | |
discovered_reqs.extend(self._resolve_one(requirement_set, req)) | |
except HashError as exc: | |
exc.req = req | |
hash_errors.append(exc) | |
if hash_errors: | |
raise hash_errors | |
def _is_upgrade_allowed(self, req): | |
# type: (InstallRequirement) -> bool | |
if self.upgrade_strategy == "to-satisfy-only": | |
return False | |
elif self.upgrade_strategy == "eager": | |
return True | |
else: | |
assert self.upgrade_strategy == "only-if-needed" | |
return req.is_direct | |
def _set_req_to_reinstall(self, req): | |
# type: (InstallRequirement) -> None | |
""" | |
Set a requirement to be installed. | |
""" | |
# Don't uninstall the conflict if doing a user install and the | |
# conflict is not a user install. | |
if not self.use_user_site or dist_in_usersite(req.satisfied_by): | |
req.should_reinstall = True | |
req.satisfied_by = None | |
def _check_skip_installed(self, req_to_install): | |
# type: (InstallRequirement) -> Optional[str] | |
"""Check if req_to_install should be skipped. | |
This will check if the req is installed, and whether we should upgrade | |
or reinstall it, taking into account all the relevant user options. | |
After calling this req_to_install will only have satisfied_by set to | |
None if the req_to_install is to be upgraded/reinstalled etc. Any | |
other value will be a dist recording the current thing installed that | |
satisfies the requirement. | |
Note that for vcs urls and the like we can't assess skipping in this | |
routine - we simply identify that we need to pull the thing down, | |
then later on it is pulled down and introspected to assess upgrade/ | |
reinstalls etc. | |
:return: A text reason for why it was skipped, or None. | |
""" | |
if self.ignore_installed: | |
return None | |
req_to_install.check_if_exists(self.use_user_site) | |
if not req_to_install.satisfied_by: | |
return None | |
if self.force_reinstall: | |
self._set_req_to_reinstall(req_to_install) | |
return None | |
if not self._is_upgrade_allowed(req_to_install): | |
if self.upgrade_strategy == "only-if-needed": | |
return 'already satisfied, skipping upgrade' | |
return 'already satisfied' | |
# Check for the possibility of an upgrade. For link-based | |
# requirements we have to pull the tree down and inspect to assess | |
# the version #, so it's handled way down. | |
if not req_to_install.link: | |
try: | |
self.finder.find_requirement(req_to_install, upgrade=True) | |
except BestVersionAlreadyInstalled: | |
# Then the best version is installed. | |
return 'already up-to-date' | |
except DistributionNotFound: | |
# No distribution found, so we squash the error. It will | |
# be raised later when we re-try later to do the install. | |
# Why don't we just raise here? | |
pass | |
self._set_req_to_reinstall(req_to_install) | |
return None | |
def _get_abstract_dist_for(self, req): | |
# type: (InstallRequirement) -> AbstractDistribution | |
"""Takes a InstallRequirement and returns a single AbstractDist \ | |
representing a prepared variant of the same. | |
""" | |
if req.editable: | |
return self.preparer.prepare_editable_requirement(req) | |
# satisfied_by is only evaluated by calling _check_skip_installed, | |
# so it must be None here. | |
assert req.satisfied_by is None | |
skip_reason = self._check_skip_installed(req) | |
if req.satisfied_by: | |
return self.preparer.prepare_installed_requirement( | |
req, skip_reason | |
) | |
upgrade_allowed = self._is_upgrade_allowed(req) | |
# We eagerly populate the link, since that's our "legacy" behavior. | |
require_hashes = self.preparer.require_hashes | |
req.populate_link(self.finder, upgrade_allowed, require_hashes) | |
abstract_dist = self.preparer.prepare_linked_requirement(req) | |
# NOTE | |
# The following portion is for determining if a certain package is | |
# going to be re-installed/upgraded or not and reporting to the user. | |
# This should probably get cleaned up in a future refactor. | |
# req.req is only avail after unpack for URL | |
# pkgs repeat check_if_exists to uninstall-on-upgrade | |
# (#14) | |
if not self.ignore_installed: | |
req.check_if_exists(self.use_user_site) | |
if req.satisfied_by: | |
should_modify = ( | |
self.upgrade_strategy != "to-satisfy-only" or | |
self.force_reinstall or | |
self.ignore_installed or | |
req.link.scheme == 'file' | |
) | |
if should_modify: | |
self._set_req_to_reinstall(req) | |
else: | |
logger.info( | |
'Requirement already satisfied (use --upgrade to upgrade):' | |
' %s', req, | |
) | |
return abstract_dist | |
def _resolve_one( | |
self, | |
requirement_set, # type: RequirementSet | |
req_to_install, # type: InstallRequirement | |
): | |
# type: (...) -> List[InstallRequirement] | |
"""Prepare a single requirements file. | |
:return: A list of additional InstallRequirements to also install. | |
""" | |
# Tell user what we are doing for this requirement: | |
# obtain (editable), skipping, processing (local url), collecting | |
# (remote url or package name) | |
if req_to_install.constraint or req_to_install.prepared: | |
return [] | |
req_to_install.prepared = True | |
# register tmp src for cleanup in case something goes wrong | |
requirement_set.reqs_to_cleanup.append(req_to_install) | |
abstract_dist = self._get_abstract_dist_for(req_to_install) | |
# Parse and return dependencies | |
dist = abstract_dist.get_pkg_resources_distribution() | |
# This will raise UnsupportedPythonVersion if the given Python | |
# version isn't compatible with the distribution's Requires-Python. | |
_check_dist_requires_python( | |
dist, version_info=self._py_version_info, | |
ignore_requires_python=self.ignore_requires_python, | |
) | |
more_reqs = [] # type: List[InstallRequirement] | |
def add_req(subreq, extras_requested): | |
sub_install_req = self._make_install_req( | |
str(subreq), | |
req_to_install, | |
) | |
parent_req_name = req_to_install.name | |
to_scan_again, add_to_parent = requirement_set.add_requirement( | |
sub_install_req, | |
parent_req_name=parent_req_name, | |
extras_requested=extras_requested, | |
) | |
if parent_req_name and add_to_parent: | |
self._discovered_dependencies[parent_req_name].append( | |
add_to_parent | |
) | |
more_reqs.extend(to_scan_again) | |
with indent_log(): | |
# We add req_to_install before its dependencies, so that we | |
# can refer to it when adding dependencies. | |
if not requirement_set.has_requirement(req_to_install.name): | |
# 'unnamed' requirements will get added here | |
# 'unnamed' requirements can only come from being directly | |
# provided by the user. | |
assert req_to_install.is_direct | |
requirement_set.add_requirement( | |
req_to_install, parent_req_name=None, | |
) | |
if not self.ignore_dependencies: | |
if req_to_install.extras: | |
logger.debug( | |
"Installing extra requirements: %r", | |
','.join(req_to_install.extras), | |
) | |
missing_requested = sorted( | |
set(req_to_install.extras) - set(dist.extras) | |
) | |
for missing in missing_requested: | |
logger.warning( | |
'%s does not provide the extra \'%s\'', | |
dist, missing | |
) | |
available_requested = sorted( | |
set(dist.extras) & set(req_to_install.extras) | |
) | |
for subreq in dist.requires(available_requested): | |
add_req(subreq, extras_requested=available_requested) | |
if not req_to_install.editable and not req_to_install.satisfied_by: | |
# XXX: --no-install leads this to report 'Successfully | |
# downloaded' for only non-editable reqs, even though we took | |
# action on them. | |
requirement_set.successfully_downloaded.append(req_to_install) | |
return more_reqs | |
def get_installation_order(self, req_set): | |
# type: (RequirementSet) -> List[InstallRequirement] | |
"""Create the installation order. | |
The installation order is topological - requirements are installed | |
before the requiring thing. We break cycles at an arbitrary point, | |
and make no other guarantees. | |
""" | |
# The current implementation, which we may change at any point | |
# installs the user specified things in the order given, except when | |
# dependencies must come earlier to achieve topological order. | |
order = [] | |
ordered_reqs = set() # type: Set[InstallRequirement] | |
def schedule(req): | |
if req.satisfied_by or req in ordered_reqs: | |
return | |
if req.constraint: | |
return | |
ordered_reqs.add(req) | |
for dep in self._discovered_dependencies[req.name]: | |
schedule(dep) | |
order.append(req) | |
for install_req in req_set.requirements.values(): | |
schedule(install_req) | |
return order |
"""Locations where we look for configs, install stuff, etc""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import os | |
import os.path | |
import platform | |
import site | |
import sys | |
import sysconfig | |
from distutils import sysconfig as distutils_sysconfig | |
from distutils.command.install import SCHEME_KEYS # type: ignore | |
from distutils.command.install import install as distutils_install_command | |
from pip._internal.models.scheme import Scheme | |
from pip._internal.utils import appdirs | |
from pip._internal.utils.compat import WINDOWS | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast | |
from pip._internal.utils.virtualenv import running_under_virtualenv | |
if MYPY_CHECK_RUNNING: | |
from typing import Dict, List, Optional, Union | |
from distutils.cmd import Command as DistutilsCommand | |
# Application Directories | |
USER_CACHE_DIR = appdirs.user_cache_dir("pip") | |
def get_major_minor_version(): | |
# type: () -> str | |
""" | |
Return the major-minor version of the current Python as a string, e.g. | |
"3.7" or "3.10". | |
""" | |
return '{}.{}'.format(*sys.version_info) | |
def get_src_prefix(): | |
# type: () -> str | |
if running_under_virtualenv(): | |
src_prefix = os.path.join(sys.prefix, 'src') | |
else: | |
# FIXME: keep src in cwd for now (it is not a temporary folder) | |
try: | |
src_prefix = os.path.join(os.getcwd(), 'src') | |
except OSError: | |
# In case the current working directory has been renamed or deleted | |
sys.exit( | |
"The folder you are executing pip from can no longer be found." | |
) | |
# under macOS + virtualenv sys.prefix is not properly resolved | |
# it is something like /path/to/python/bin/.. | |
return os.path.abspath(src_prefix) | |
# FIXME doesn't account for venv linked to global site-packages | |
site_packages = sysconfig.get_path("purelib") # type: Optional[str] | |
# This is because of a bug in PyPy's sysconfig module, see | |
# https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths | |
# for more information. | |
if platform.python_implementation().lower() == "pypy": | |
site_packages = distutils_sysconfig.get_python_lib() | |
try: | |
# Use getusersitepackages if this is present, as it ensures that the | |
# value is initialised properly. | |
user_site = site.getusersitepackages() | |
except AttributeError: | |
user_site = site.USER_SITE | |
if WINDOWS: | |
bin_py = os.path.join(sys.prefix, 'Scripts') | |
bin_user = os.path.join(user_site, 'Scripts') | |
# buildout uses 'bin' on Windows too? | |
if not os.path.exists(bin_py): | |
bin_py = os.path.join(sys.prefix, 'bin') | |
bin_user = os.path.join(user_site, 'bin') | |
else: | |
bin_py = os.path.join(sys.prefix, 'bin') | |
bin_user = os.path.join(user_site, 'bin') | |
# Forcing to use /usr/local/bin for standard macOS framework installs | |
# Also log to ~/Library/Logs/ for use with the Console.app log viewer | |
if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': | |
bin_py = '/usr/local/bin' | |
def distutils_scheme( | |
dist_name, user=False, home=None, root=None, isolated=False, prefix=None | |
): | |
# type:(str, bool, str, str, bool, str) -> Dict[str, str] | |
""" | |
Return a distutils install scheme | |
""" | |
from distutils.dist import Distribution | |
dist_args = {'name': dist_name} # type: Dict[str, Union[str, List[str]]] | |
if isolated: | |
dist_args["script_args"] = ["--no-user-cfg"] | |
d = Distribution(dist_args) | |
d.parse_config_files() | |
obj = None # type: Optional[DistutilsCommand] | |
obj = d.get_command_obj('install', create=True) | |
assert obj is not None | |
i = cast(distutils_install_command, obj) | |
# NOTE: setting user or home has the side-effect of creating the home dir | |
# or user base for installations during finalize_options() | |
# ideally, we'd prefer a scheme class that has no side-effects. | |
assert not (user and prefix), "user={} prefix={}".format(user, prefix) | |
assert not (home and prefix), "home={} prefix={}".format(home, prefix) | |
i.user = user or i.user | |
if user or home: | |
i.prefix = "" | |
i.prefix = prefix or i.prefix | |
i.home = home or i.home | |
i.root = root or i.root | |
i.finalize_options() | |
scheme = {} | |
for key in SCHEME_KEYS: | |
scheme[key] = getattr(i, 'install_' + key) | |
# install_lib specified in setup.cfg should install *everything* | |
# into there (i.e. it takes precedence over both purelib and | |
# platlib). Note, i.install_lib is *always* set after | |
# finalize_options(); we only want to override here if the user | |
# has explicitly requested it hence going back to the config | |
if 'install_lib' in d.get_option_dict('install'): | |
scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) | |
if running_under_virtualenv(): | |
scheme['headers'] = os.path.join( | |
sys.prefix, | |
'include', | |
'site', | |
'python{}'.format(get_major_minor_version()), | |
dist_name, | |
) | |
if root is not None: | |
path_no_drive = os.path.splitdrive( | |
os.path.abspath(scheme["headers"]))[1] | |
scheme["headers"] = os.path.join( | |
root, | |
path_no_drive[1:], | |
) | |
return scheme | |
def get_scheme( | |
dist_name, # type: str | |
user=False, # type: bool | |
home=None, # type: Optional[str] | |
root=None, # type: Optional[str] | |
isolated=False, # type: bool | |
prefix=None, # type: Optional[str] | |
): | |
# type: (...) -> Scheme | |
""" | |
Get the "scheme" corresponding to the input parameters. The distutils | |
documentation provides the context for the available schemes: | |
https://docs.python.org/3/install/index.html#alternate-installation | |
:param dist_name: the name of the package to retrieve the scheme for, used | |
in the headers scheme path | |
:param user: indicates to use the "user" scheme | |
:param home: indicates to use the "home" scheme and provides the base | |
directory for the same | |
:param root: root under which other directories are re-based | |
:param isolated: equivalent to --no-user-cfg, i.e. do not consider | |
~/.pydistutils.cfg (posix) or ~/pydistutils.cfg (non-posix) for | |
scheme paths | |
:param prefix: indicates to use the "prefix" scheme and provides the | |
base directory for the same | |
""" | |
scheme = distutils_scheme( | |
dist_name, user, home, root, isolated, prefix | |
) | |
return Scheme( | |
platlib=scheme["platlib"], | |
purelib=scheme["purelib"], | |
headers=scheme["headers"], | |
scripts=scheme["scripts"], | |
data=scheme["data"], | |
) |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, List | |
def main(args=None): | |
# type: (Optional[List[str]]) -> int | |
"""This is preserved for old console scripts that may still be referencing | |
it. | |
For additional details, see https://github.com/pypa/pip/issues/7498. | |
""" | |
from pip._internal.utils.entrypoints import _wrapper | |
return _wrapper(args) |
"""A package that contains models that represent entities. | |
""" |
from pip._vendor.packaging.version import parse as parse_version | |
from pip._internal.utils.models import KeyBasedCompareMixin | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from pip._vendor.packaging.version import _BaseVersion | |
from pip._internal.models.link import Link | |
class InstallationCandidate(KeyBasedCompareMixin): | |
"""Represents a potential "candidate" for installation. | |
""" | |
def __init__(self, name, version, link): | |
# type: (str, str, Link) -> None | |
self.name = name | |
self.version = parse_version(version) # type: _BaseVersion | |
self.link = link | |
super(InstallationCandidate, self).__init__( | |
key=(self.name, self.version, self.link), | |
defining_class=InstallationCandidate | |
) | |
def __repr__(self): | |
# type: () -> str | |
return "<InstallationCandidate({!r}, {!r}, {!r})>".format( | |
self.name, self.version, self.link, | |
) | |
def __str__(self): | |
# type: () -> str | |
return '{!r} candidate (version {} at {})'.format( | |
self.name, self.version, self.link, | |
) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal.exceptions import CommandError | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Set, FrozenSet | |
class FormatControl(object): | |
"""Helper for managing formats from which a package can be installed. | |
""" | |
def __init__(self, no_binary=None, only_binary=None): | |
# type: (Optional[Set[str]], Optional[Set[str]]) -> None | |
if no_binary is None: | |
no_binary = set() | |
if only_binary is None: | |
only_binary = set() | |
self.no_binary = no_binary | |
self.only_binary = only_binary | |
def __eq__(self, other): | |
# type: (object) -> bool | |
return self.__dict__ == other.__dict__ | |
def __ne__(self, other): | |
# type: (object) -> bool | |
return not self.__eq__(other) | |
def __repr__(self): | |
# type: () -> str | |
return "{}({}, {})".format( | |
self.__class__.__name__, | |
self.no_binary, | |
self.only_binary | |
) | |
@staticmethod | |
def handle_mutual_excludes(value, target, other): | |
# type: (str, Optional[Set[str]], Optional[Set[str]]) -> None | |
if value.startswith('-'): | |
raise CommandError( | |
"--no-binary / --only-binary option requires 1 argument." | |
) | |
new = value.split(',') | |
while ':all:' in new: | |
other.clear() | |
target.clear() | |
target.add(':all:') | |
del new[:new.index(':all:') + 1] | |
# Without a none, we want to discard everything as :all: covers it | |
if ':none:' not in new: | |
return | |
for name in new: | |
if name == ':none:': | |
target.clear() | |
continue | |
name = canonicalize_name(name) | |
other.discard(name) | |
target.add(name) | |
def get_allowed_formats(self, canonical_name): | |
# type: (str) -> FrozenSet[str] | |
result = {"binary", "source"} | |
if canonical_name in self.only_binary: | |
result.discard('source') | |
elif canonical_name in self.no_binary: | |
result.discard('binary') | |
elif ':all:' in self.only_binary: | |
result.discard('source') | |
elif ':all:' in self.no_binary: | |
result.discard('binary') | |
return frozenset(result) | |
def disallow_binaries(self): | |
# type: () -> None | |
self.handle_mutual_excludes( | |
':all:', self.no_binary, self.only_binary, | |
) |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
class PackageIndex(object): | |
"""Represents a Package Index and provides easier access to endpoints | |
""" | |
def __init__(self, url, file_storage_domain): | |
# type: (str, str) -> None | |
super(PackageIndex, self).__init__() | |
self.url = url | |
self.netloc = urllib_parse.urlsplit(url).netloc | |
self.simple_url = self._url_for_path('simple') | |
self.pypi_url = self._url_for_path('pypi') | |
# This is part of a temporary hack used to block installs of PyPI | |
# packages which depend on external urls only necessary until PyPI can | |
# block such packages themselves | |
self.file_storage_domain = file_storage_domain | |
def _url_for_path(self, path): | |
# type: (str) -> str | |
return urllib_parse.urljoin(self.url, path) | |
PyPI = PackageIndex( | |
'https://pypi.org/', file_storage_domain='files.pythonhosted.org' | |
) | |
TestPyPI = PackageIndex( | |
'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' | |
) |
import os | |
import posixpath | |
import re | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.utils.filetypes import WHEEL_EXTENSION | |
from pip._internal.utils.misc import ( | |
redact_auth_from_url, | |
split_auth_from_netloc, | |
splitext, | |
) | |
from pip._internal.utils.models import KeyBasedCompareMixin | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url, url_to_path | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Text, Tuple, Union | |
from pip._internal.index.collector import HTMLPage | |
from pip._internal.utils.hashes import Hashes | |
class Link(KeyBasedCompareMixin): | |
"""Represents a parsed link from a Package Index's simple URL | |
""" | |
def __init__( | |
self, | |
url, # type: str | |
comes_from=None, # type: Optional[Union[str, HTMLPage]] | |
requires_python=None, # type: Optional[str] | |
yanked_reason=None, # type: Optional[Text] | |
): | |
# type: (...) -> None | |
""" | |
:param url: url of the resource pointed to (href of the link) | |
:param comes_from: instance of HTMLPage where the link was found, | |
or string. | |
:param requires_python: String containing the `Requires-Python` | |
metadata field, specified in PEP 345. This may be specified by | |
a data-requires-python attribute in the HTML link tag, as | |
described in PEP 503. | |
:param yanked_reason: the reason the file has been yanked, if the | |
file has been yanked, or None if the file hasn't been yanked. | |
This is the value of the "data-yanked" attribute, if present, in | |
a simple repository HTML link. If the file has been yanked but | |
no reason was provided, this should be the empty string. See | |
PEP 592 for more information and the specification. | |
""" | |
# url can be a UNC windows share | |
if url.startswith('\\\\'): | |
url = path_to_url(url) | |
self._parsed_url = urllib_parse.urlsplit(url) | |
# Store the url as a private attribute to prevent accidentally | |
# trying to set a new value. | |
self._url = url | |
self.comes_from = comes_from | |
self.requires_python = requires_python if requires_python else None | |
self.yanked_reason = yanked_reason | |
super(Link, self).__init__(key=url, defining_class=Link) | |
def __str__(self): | |
# type: () -> str | |
if self.requires_python: | |
rp = ' (requires-python:%s)' % self.requires_python | |
else: | |
rp = '' | |
if self.comes_from: | |
return '%s (from %s)%s' % (redact_auth_from_url(self._url), | |
self.comes_from, rp) | |
else: | |
return redact_auth_from_url(str(self._url)) | |
def __repr__(self): | |
# type: () -> str | |
return '<Link %s>' % self | |
@property | |
def url(self): | |
# type: () -> str | |
return self._url | |
@property | |
def filename(self): | |
# type: () -> str | |
path = self.path.rstrip('/') | |
name = posixpath.basename(path) | |
if not name: | |
# Make sure we don't leak auth information if the netloc | |
# includes a username and password. | |
netloc, user_pass = split_auth_from_netloc(self.netloc) | |
return netloc | |
name = urllib_parse.unquote(name) | |
assert name, ('URL %r produced no filename' % self._url) | |
return name | |
@property | |
def file_path(self): | |
# type: () -> str | |
return url_to_path(self.url) | |
@property | |
def scheme(self): | |
# type: () -> str | |
return self._parsed_url.scheme | |
@property | |
def netloc(self): | |
# type: () -> str | |
""" | |
This can contain auth information. | |
""" | |
return self._parsed_url.netloc | |
@property | |
def path(self): | |
# type: () -> str | |
return urllib_parse.unquote(self._parsed_url.path) | |
def splitext(self): | |
# type: () -> Tuple[str, str] | |
return splitext(posixpath.basename(self.path.rstrip('/'))) | |
@property | |
def ext(self): | |
# type: () -> str | |
return self.splitext()[1] | |
@property | |
def url_without_fragment(self): | |
# type: () -> str | |
scheme, netloc, path, query, fragment = self._parsed_url | |
return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) | |
_egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') | |
@property | |
def egg_fragment(self): | |
# type: () -> Optional[str] | |
match = self._egg_fragment_re.search(self._url) | |
if not match: | |
return None | |
return match.group(1) | |
_subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') | |
@property | |
def subdirectory_fragment(self): | |
# type: () -> Optional[str] | |
match = self._subdirectory_fragment_re.search(self._url) | |
if not match: | |
return None | |
return match.group(1) | |
_hash_re = re.compile( | |
r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' | |
) | |
@property | |
def hash(self): | |
# type: () -> Optional[str] | |
match = self._hash_re.search(self._url) | |
if match: | |
return match.group(2) | |
return None | |
@property | |
def hash_name(self): | |
# type: () -> Optional[str] | |
match = self._hash_re.search(self._url) | |
if match: | |
return match.group(1) | |
return None | |
@property | |
def show_url(self): | |
# type: () -> str | |
return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0]) | |
@property | |
def is_file(self): | |
# type: () -> bool | |
return self.scheme == 'file' | |
def is_existing_dir(self): | |
# type: () -> bool | |
return self.is_file and os.path.isdir(self.file_path) | |
@property | |
def is_wheel(self): | |
# type: () -> bool | |
return self.ext == WHEEL_EXTENSION | |
@property | |
def is_vcs(self): | |
# type: () -> bool | |
from pip._internal.vcs import vcs | |
return self.scheme in vcs.all_schemes | |
@property | |
def is_yanked(self): | |
# type: () -> bool | |
return self.yanked_reason is not None | |
@property | |
def has_hash(self): | |
# type: () -> bool | |
return self.hash_name is not None | |
def is_hash_allowed(self, hashes): | |
# type: (Optional[Hashes]) -> bool | |
""" | |
Return True if the link has a hash and it is allowed. | |
""" | |
if hashes is None or not self.has_hash: | |
return False | |
# Assert non-None so mypy knows self.hash_name and self.hash are str. | |
assert self.hash_name is not None | |
assert self.hash is not None | |
return hashes.is_hash_allowed(self.hash_name, hex_digest=self.hash) |
""" | |
For types associated with installation schemes. | |
For a general overview of available schemes and their context, see | |
https://docs.python.org/3/install/index.html#alternate-installation. | |
""" | |
class Scheme(object): | |
"""A Scheme holds paths which are used as the base directories for | |
artifacts associated with a Python package. | |
""" | |
def __init__( | |
self, | |
platlib, # type: str | |
purelib, # type: str | |
headers, # type: str | |
scripts, # type: str | |
data, # type: str | |
): | |
self.platlib = platlib | |
self.purelib = purelib | |
self.headers = headers | |
self.scripts = scripts | |
self.data = data |
import itertools | |
import logging | |
import os | |
import posixpath | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.models.index import PyPI | |
from pip._internal.utils.compat import has_tls | |
from pip._internal.utils.misc import normalize_path, redact_auth_from_url | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List | |
logger = logging.getLogger(__name__) | |
class SearchScope(object): | |
""" | |
Encapsulates the locations that pip is configured to search. | |
""" | |
@classmethod | |
def create( | |
cls, | |
find_links, # type: List[str] | |
index_urls, # type: List[str] | |
): | |
# type: (...) -> SearchScope | |
""" | |
Create a SearchScope object after normalizing the `find_links`. | |
""" | |
# Build find_links. If an argument starts with ~, it may be | |
# a local file relative to a home directory. So try normalizing | |
# it and if it exists, use the normalized version. | |
# This is deliberately conservative - it might be fine just to | |
# blindly normalize anything starting with a ~... | |
built_find_links = [] # type: List[str] | |
for link in find_links: | |
if link.startswith('~'): | |
new_link = normalize_path(link) | |
if os.path.exists(new_link): | |
link = new_link | |
built_find_links.append(link) | |
# If we don't have TLS enabled, then WARN if anyplace we're looking | |
# relies on TLS. | |
if not has_tls(): | |
for link in itertools.chain(index_urls, built_find_links): | |
parsed = urllib_parse.urlparse(link) | |
if parsed.scheme == 'https': | |
logger.warning( | |
'pip is configured with locations that require ' | |
'TLS/SSL, however the ssl module in Python is not ' | |
'available.' | |
) | |
break | |
return cls( | |
find_links=built_find_links, | |
index_urls=index_urls, | |
) | |
def __init__( | |
self, | |
find_links, # type: List[str] | |
index_urls, # type: List[str] | |
): | |
# type: (...) -> None | |
self.find_links = find_links | |
self.index_urls = index_urls | |
def get_formatted_locations(self): | |
# type: () -> str | |
lines = [] | |
if self.index_urls and self.index_urls != [PyPI.simple_url]: | |
lines.append( | |
'Looking in indexes: {}'.format(', '.join( | |
redact_auth_from_url(url) for url in self.index_urls)) | |
) | |
if self.find_links: | |
lines.append( | |
'Looking in links: {}'.format(', '.join( | |
redact_auth_from_url(url) for url in self.find_links)) | |
) | |
return '\n'.join(lines) | |
def get_index_urls_locations(self, project_name): | |
# type: (str) -> List[str] | |
"""Returns the locations found via self.index_urls | |
Checks the url_name on the main (first in the list) index and | |
use this url_name to produce all locations | |
""" | |
def mkurl_pypi_url(url): | |
# type: (str) -> str | |
loc = posixpath.join( | |
url, | |
urllib_parse.quote(canonicalize_name(project_name))) | |
# For maximum compatibility with easy_install, ensure the path | |
# ends in a trailing slash. Although this isn't in the spec | |
# (and PyPI can handle it without the slash) some other index | |
# implementations might break if they relied on easy_install's | |
# behavior. | |
if not loc.endswith('/'): | |
loc = loc + '/' | |
return loc | |
return [mkurl_pypi_url(url) for url in self.index_urls] |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional | |
from pip._internal.models.format_control import FormatControl | |
class SelectionPreferences(object): | |
""" | |
Encapsulates the candidate selection preferences for downloading | |
and installing files. | |
""" | |
# Don't include an allow_yanked default value to make sure each call | |
# site considers whether yanked releases are allowed. This also causes | |
# that decision to be made explicit in the calling code, which helps | |
# people when reading the code. | |
def __init__( | |
self, | |
allow_yanked, # type: bool | |
allow_all_prereleases=False, # type: bool | |
format_control=None, # type: Optional[FormatControl] | |
prefer_binary=False, # type: bool | |
ignore_requires_python=None, # type: Optional[bool] | |
): | |
# type: (...) -> None | |
"""Create a SelectionPreferences object. | |
:param allow_yanked: Whether files marked as yanked (in the sense | |
of PEP 592) are permitted to be candidates for install. | |
:param format_control: A FormatControl object or None. Used to control | |
the selection of source packages / binary packages when consulting | |
the index and links. | |
:param prefer_binary: Whether to prefer an old, but valid, binary | |
dist over a new source dist. | |
:param ignore_requires_python: Whether to ignore incompatible | |
"Requires-Python" values in links. Defaults to False. | |
""" | |
if ignore_requires_python is None: | |
ignore_requires_python = False | |
self.allow_yanked = allow_yanked | |
self.allow_all_prereleases = allow_all_prereleases | |
self.format_control = format_control | |
self.prefer_binary = prefer_binary | |
self.ignore_requires_python = ignore_requires_python |
import sys | |
from pip._internal.pep425tags import get_supported, version_info_to_nodot | |
from pip._internal.utils.misc import normalize_version_info | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional, Tuple | |
from pip._vendor.packaging.tags import Tag | |
class TargetPython(object): | |
""" | |
Encapsulates the properties of a Python interpreter one is targeting | |
for a package install, download, etc. | |
""" | |
def __init__( | |
self, | |
platform=None, # type: Optional[str] | |
py_version_info=None, # type: Optional[Tuple[int, ...]] | |
abi=None, # type: Optional[str] | |
implementation=None, # type: Optional[str] | |
): | |
# type: (...) -> None | |
""" | |
:param platform: A string or None. If None, searches for packages | |
that are supported by the current system. Otherwise, will find | |
packages that can be built on the platform passed in. These | |
packages will only be downloaded for distribution: they will | |
not be built locally. | |
:param py_version_info: An optional tuple of ints representing the | |
Python version information to use (e.g. `sys.version_info[:3]`). | |
This can have length 1, 2, or 3 when provided. | |
:param abi: A string or None. This is passed to pep425tags.py's | |
get_supported() function as is. | |
:param implementation: A string or None. This is passed to | |
pep425tags.py's get_supported() function as is. | |
""" | |
# Store the given py_version_info for when we call get_supported(). | |
self._given_py_version_info = py_version_info | |
if py_version_info is None: | |
py_version_info = sys.version_info[:3] | |
else: | |
py_version_info = normalize_version_info(py_version_info) | |
py_version = '.'.join(map(str, py_version_info[:2])) | |
self.abi = abi | |
self.implementation = implementation | |
self.platform = platform | |
self.py_version = py_version | |
self.py_version_info = py_version_info | |
# This is used to cache the return value of get_tags(). | |
self._valid_tags = None # type: Optional[List[Tag]] | |
def format_given(self): | |
# type: () -> str | |
""" | |
Format the given, non-None attributes for display. | |
""" | |
display_version = None | |
if self._given_py_version_info is not None: | |
display_version = '.'.join( | |
str(part) for part in self._given_py_version_info | |
) | |
key_values = [ | |
('platform', self.platform), | |
('version_info', display_version), | |
('abi', self.abi), | |
('implementation', self.implementation), | |
] | |
return ' '.join( | |
'{}={!r}'.format(key, value) for key, value in key_values | |
if value is not None | |
) | |
def get_tags(self): | |
# type: () -> List[Tag] | |
""" | |
Return the supported PEP 425 tags to check wheel candidates against. | |
The tags are returned in order of preference (most preferred first). | |
""" | |
if self._valid_tags is None: | |
# Pass versions=None if no py_version_info was given since | |
# versions=None uses special default logic. | |
py_version_info = self._given_py_version_info | |
if py_version_info is None: | |
version = None | |
else: | |
version = version_info_to_nodot(py_version_info) | |
tags = get_supported( | |
version=version, | |
platform=self.platform, | |
abi=self.abi, | |
impl=self.implementation, | |
) | |
self._valid_tags = tags | |
return self._valid_tags |
"""Represents a wheel file and provides access to the various parts of the | |
name that have meaning. | |
""" | |
import re | |
from pip._vendor.packaging.tags import Tag | |
from pip._internal.exceptions import InvalidWheelFilename | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List | |
class Wheel(object): | |
"""A wheel file""" | |
wheel_file_re = re.compile( | |
r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?)) | |
((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?) | |
\.whl|\.dist-info)$""", | |
re.VERBOSE | |
) | |
def __init__(self, filename): | |
# type: (str) -> None | |
""" | |
:raises InvalidWheelFilename: when the filename is invalid for a wheel | |
""" | |
wheel_info = self.wheel_file_re.match(filename) | |
if not wheel_info: | |
raise InvalidWheelFilename( | |
"%s is not a valid wheel filename." % filename | |
) | |
self.filename = filename | |
self.name = wheel_info.group('name').replace('_', '-') | |
# we'll assume "_" means "-" due to wheel naming scheme | |
# (https://github.com/pypa/pip/issues/1150) | |
self.version = wheel_info.group('ver').replace('_', '-') | |
self.build_tag = wheel_info.group('build') | |
self.pyversions = wheel_info.group('pyver').split('.') | |
self.abis = wheel_info.group('abi').split('.') | |
self.plats = wheel_info.group('plat').split('.') | |
# All the tag combinations from this file | |
self.file_tags = { | |
Tag(x, y, z) for x in self.pyversions | |
for y in self.abis for z in self.plats | |
} | |
def get_formatted_file_tags(self): | |
# type: () -> List[str] | |
"""Return the wheel's tags as a sorted list of strings.""" | |
return sorted(str(tag) for tag in self.file_tags) | |
def support_index_min(self, tags): | |
# type: (List[Tag]) -> int | |
"""Return the lowest index that one of the wheel's file_tag combinations | |
achieves in the given list of supported tags. | |
For example, if there are 8 supported tags and one of the file tags | |
is first in the list, then return 0. | |
:param tags: the PEP 425 tags to check the wheel against, in order | |
with most preferred first. | |
:raises ValueError: If none of the wheel's file tags match one of | |
the supported tags. | |
""" | |
return min(tags.index(tag) for tag in self.file_tags if tag in tags) | |
def supported(self, tags): | |
# type: (List[Tag]) -> bool | |
"""Return whether the wheel is compatible with one of the given tags. | |
:param tags: the PEP 425 tags to check the wheel against. | |
""" | |
return not self.file_tags.isdisjoint(tags) |
"""Contains purely network-related utilities. | |
""" |
"""Network Authentication Helpers | |
Contains interface (MultiDomainBasicAuth) and associated glue code for | |
providing credentials in the context of network requests. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import logging | |
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth | |
from pip._vendor.requests.utils import get_netrc_auth | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.utils.misc import ( | |
ask, | |
ask_input, | |
ask_password, | |
remove_auth_from_url, | |
split_auth_netloc_from_url, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from optparse import Values | |
from typing import Dict, Optional, Tuple | |
from pip._internal.vcs.versioncontrol import AuthInfo | |
Credentials = Tuple[str, str, str] | |
logger = logging.getLogger(__name__) | |
try: | |
import keyring # noqa | |
except ImportError: | |
keyring = None | |
except Exception as exc: | |
logger.warning( | |
"Keyring is skipped due to an exception: %s", str(exc), | |
) | |
keyring = None | |
def get_keyring_auth(url, username): | |
"""Return the tuple auth for a given url from keyring.""" | |
if not url or not keyring: | |
return None | |
try: | |
try: | |
get_credential = keyring.get_credential | |
except AttributeError: | |
pass | |
else: | |
logger.debug("Getting credentials from keyring for %s", url) | |
cred = get_credential(url, username) | |
if cred is not None: | |
return cred.username, cred.password | |
return None | |
if username: | |
logger.debug("Getting password from keyring for %s", url) | |
password = keyring.get_password(url, username) | |
if password: | |
return username, password | |
except Exception as exc: | |
logger.warning( | |
"Keyring is skipped due to an exception: %s", str(exc), | |
) | |
class MultiDomainBasicAuth(AuthBase): | |
def __init__(self, prompting=True, index_urls=None): | |
# type: (bool, Optional[Values]) -> None | |
self.prompting = prompting | |
self.index_urls = index_urls | |
self.passwords = {} # type: Dict[str, AuthInfo] | |
# When the user is prompted to enter credentials and keyring is | |
# available, we will offer to save them. If the user accepts, | |
# this value is set to the credentials they entered. After the | |
# request authenticates, the caller should call | |
# ``save_credentials`` to save these. | |
self._credentials_to_save = None # type: Optional[Credentials] | |
def _get_index_url(self, url): | |
"""Return the original index URL matching the requested URL. | |
Cached or dynamically generated credentials may work against | |
the original index URL rather than just the netloc. | |
The provided url should have had its username and password | |
removed already. If the original index url had credentials then | |
they will be included in the return value. | |
Returns None if no matching index was found, or if --no-index | |
was specified by the user. | |
""" | |
if not url or not self.index_urls: | |
return None | |
for u in self.index_urls: | |
prefix = remove_auth_from_url(u).rstrip("/") + "/" | |
if url.startswith(prefix): | |
return u | |
def _get_new_credentials(self, original_url, allow_netrc=True, | |
allow_keyring=True): | |
"""Find and return credentials for the specified URL.""" | |
# Split the credentials and netloc from the url. | |
url, netloc, url_user_password = split_auth_netloc_from_url( | |
original_url, | |
) | |
# Start with the credentials embedded in the url | |
username, password = url_user_password | |
if username is not None and password is not None: | |
logger.debug("Found credentials in url for %s", netloc) | |
return url_user_password | |
# Find a matching index url for this request | |
index_url = self._get_index_url(url) | |
if index_url: | |
# Split the credentials from the url. | |
index_info = split_auth_netloc_from_url(index_url) | |
if index_info: | |
index_url, _, index_url_user_password = index_info | |
logger.debug("Found index url %s", index_url) | |
# If an index URL was found, try its embedded credentials | |
if index_url and index_url_user_password[0] is not None: | |
username, password = index_url_user_password | |
if username is not None and password is not None: | |
logger.debug("Found credentials in index url for %s", netloc) | |
return index_url_user_password | |
# Get creds from netrc if we still don't have them | |
if allow_netrc: | |
netrc_auth = get_netrc_auth(original_url) | |
if netrc_auth: | |
logger.debug("Found credentials in netrc for %s", netloc) | |
return netrc_auth | |
# If we don't have a password and keyring is available, use it. | |
if allow_keyring: | |
# The index url is more specific than the netloc, so try it first | |
kr_auth = ( | |
get_keyring_auth(index_url, username) or | |
get_keyring_auth(netloc, username) | |
) | |
if kr_auth: | |
logger.debug("Found credentials in keyring for %s", netloc) | |
return kr_auth | |
return username, password | |
def _get_url_and_credentials(self, original_url): | |
"""Return the credentials to use for the provided URL. | |
If allowed, netrc and keyring may be used to obtain the | |
correct credentials. | |
Returns (url_without_credentials, username, password). Note | |
that even if the original URL contains credentials, this | |
function may return a different username and password. | |
""" | |
url, netloc, _ = split_auth_netloc_from_url(original_url) | |
# Use any stored credentials that we have for this netloc | |
username, password = self.passwords.get(netloc, (None, None)) | |
if username is None and password is None: | |
# No stored credentials. Acquire new credentials without prompting | |
# the user. (e.g. from netrc, keyring, or the URL itself) | |
username, password = self._get_new_credentials(original_url) | |
if username is not None or password is not None: | |
# Convert the username and password if they're None, so that | |
# this netloc will show up as "cached" in the conditional above. | |
# Further, HTTPBasicAuth doesn't accept None, so it makes sense to | |
# cache the value that is going to be used. | |
username = username or "" | |
password = password or "" | |
# Store any acquired credentials. | |
self.passwords[netloc] = (username, password) | |
assert ( | |
# Credentials were found | |
(username is not None and password is not None) or | |
# Credentials were not found | |
(username is None and password is None) | |
), "Could not load credentials from url: {}".format(original_url) | |
return url, username, password | |
def __call__(self, req): | |
# Get credentials for this request | |
url, username, password = self._get_url_and_credentials(req.url) | |
# Set the url of the request to the url without any credentials | |
req.url = url | |
if username is not None and password is not None: | |
# Send the basic auth with this request | |
req = HTTPBasicAuth(username, password)(req) | |
# Attach a hook to handle 401 responses | |
req.register_hook("response", self.handle_401) | |
return req | |
# Factored out to allow for easy patching in tests | |
def _prompt_for_password(self, netloc): | |
username = ask_input("User for %s: " % netloc) | |
if not username: | |
return None, None | |
auth = get_keyring_auth(netloc, username) | |
if auth: | |
return auth[0], auth[1], False | |
password = ask_password("Password: ") | |
return username, password, True | |
# Factored out to allow for easy patching in tests | |
def _should_save_password_to_keyring(self): | |
if not keyring: | |
return False | |
return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y" | |
def handle_401(self, resp, **kwargs): | |
# We only care about 401 responses, anything else we want to just | |
# pass through the actual response | |
if resp.status_code != 401: | |
return resp | |
# We are not able to prompt the user so simply return the response | |
if not self.prompting: | |
return resp | |
parsed = urllib_parse.urlparse(resp.url) | |
# Prompt the user for a new username and password | |
username, password, save = self._prompt_for_password(parsed.netloc) | |
# Store the new username and password to use for future requests | |
self._credentials_to_save = None | |
if username is not None and password is not None: | |
self.passwords[parsed.netloc] = (username, password) | |
# Prompt to save the password to keyring | |
if save and self._should_save_password_to_keyring(): | |
self._credentials_to_save = (parsed.netloc, username, password) | |
# Consume content and release the original connection to allow our new | |
# request to reuse the same one. | |
resp.content | |
resp.raw.release_conn() | |
# Add our new username and password to the request | |
req = HTTPBasicAuth(username or "", password or "")(resp.request) | |
req.register_hook("response", self.warn_on_401) | |
# On successful request, save the credentials that were used to | |
# keyring. (Note that if the user responded "no" above, this member | |
# is not set and nothing will be saved.) | |
if self._credentials_to_save: | |
req.register_hook("response", self.save_credentials) | |
# Send our new request | |
new_resp = resp.connection.send(req, **kwargs) | |
new_resp.history.append(resp) | |
return new_resp | |
def warn_on_401(self, resp, **kwargs): | |
"""Response callback to warn about incorrect credentials.""" | |
if resp.status_code == 401: | |
logger.warning( | |
'401 Error, Credentials not correct for %s', resp.request.url, | |
) | |
def save_credentials(self, resp, **kwargs): | |
"""Response callback to save credentials on success.""" | |
assert keyring is not None, "should never reach here without keyring" | |
if not keyring: | |
return | |
creds = self._credentials_to_save | |
self._credentials_to_save = None | |
if creds and resp.status_code < 400: | |
try: | |
logger.info('Saving credentials to keyring') | |
keyring.set_password(*creds) | |
except Exception: | |
logger.exception('Failed to save credentials') |
"""HTTP cache implementation. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import os | |
from contextlib import contextmanager | |
from pip._vendor.cachecontrol.cache import BaseCache | |
from pip._vendor.cachecontrol.caches import FileCache | |
from pip._vendor.requests.models import Response | |
from pip._internal.utils.filesystem import adjacent_tmp_file, replace | |
from pip._internal.utils.misc import ensure_dir | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional | |
def is_from_cache(response): | |
# type: (Response) -> bool | |
return getattr(response, "from_cache", False) | |
@contextmanager | |
def suppressed_cache_errors(): | |
"""If we can't access the cache then we can just skip caching and process | |
requests as if caching wasn't enabled. | |
""" | |
try: | |
yield | |
except (OSError, IOError): | |
pass | |
class SafeFileCache(BaseCache): | |
""" | |
A file based cache which is safe to use even when the target directory may | |
not be accessible or writable. | |
""" | |
def __init__(self, directory): | |
# type: (str) -> None | |
assert directory is not None, "Cache directory must not be None." | |
super(SafeFileCache, self).__init__() | |
self.directory = directory | |
def _get_cache_path(self, name): | |
# type: (str) -> str | |
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our | |
# class for backwards-compatibility and to avoid using a non-public | |
# method. | |
hashed = FileCache.encode(name) | |
parts = list(hashed[:5]) + [hashed] | |
return os.path.join(self.directory, *parts) | |
def get(self, key): | |
# type: (str) -> Optional[bytes] | |
path = self._get_cache_path(key) | |
with suppressed_cache_errors(): | |
with open(path, 'rb') as f: | |
return f.read() | |
def set(self, key, value): | |
# type: (str, bytes) -> None | |
path = self._get_cache_path(key) | |
with suppressed_cache_errors(): | |
ensure_dir(os.path.dirname(path)) | |
with adjacent_tmp_file(path) as f: | |
f.write(value) | |
replace(f.name, path) | |
def delete(self, key): | |
# type: (str) -> None | |
path = self._get_cache_path(key) | |
with suppressed_cache_errors(): | |
os.remove(path) |
"""Download files with progress indicators. | |
""" | |
import cgi | |
import logging | |
import mimetypes | |
import os | |
from pip._vendor import requests | |
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE | |
from pip._internal.models.index import PyPI | |
from pip._internal.network.cache import is_from_cache | |
from pip._internal.network.utils import response_chunks | |
from pip._internal.utils.misc import ( | |
format_size, | |
redact_auth_from_url, | |
splitext, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.ui import DownloadProgressProvider | |
if MYPY_CHECK_RUNNING: | |
from typing import Iterable, Optional | |
from pip._vendor.requests.models import Response | |
from pip._internal.models.link import Link | |
from pip._internal.network.session import PipSession | |
logger = logging.getLogger(__name__) | |
def _get_http_response_size(resp): | |
# type: (Response) -> Optional[int] | |
try: | |
return int(resp.headers['content-length']) | |
except (ValueError, KeyError, TypeError): | |
return None | |
def _prepare_download( | |
resp, # type: Response | |
link, # type: Link | |
progress_bar # type: str | |
): | |
# type: (...) -> Iterable[bytes] | |
total_length = _get_http_response_size(resp) | |
if link.netloc == PyPI.file_storage_domain: | |
url = link.show_url | |
else: | |
url = link.url_without_fragment | |
logged_url = redact_auth_from_url(url) | |
if total_length: | |
logged_url = '{} ({})'.format(logged_url, format_size(total_length)) | |
if is_from_cache(resp): | |
logger.info("Using cached %s", logged_url) | |
else: | |
logger.info("Downloading %s", logged_url) | |
if logger.getEffectiveLevel() > logging.INFO: | |
show_progress = False | |
elif is_from_cache(resp): | |
show_progress = False | |
elif not total_length: | |
show_progress = True | |
elif total_length > (40 * 1000): | |
show_progress = True | |
else: | |
show_progress = False | |
chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) | |
if not show_progress: | |
return chunks | |
return DownloadProgressProvider( | |
progress_bar, max=total_length | |
)(chunks) | |
def sanitize_content_filename(filename): | |
# type: (str) -> str | |
""" | |
Sanitize the "filename" value from a Content-Disposition header. | |
""" | |
return os.path.basename(filename) | |
def parse_content_disposition(content_disposition, default_filename): | |
# type: (str, str) -> str | |
""" | |
Parse the "filename" value from a Content-Disposition header, and | |
return the default filename if the result is empty. | |
""" | |
_type, params = cgi.parse_header(content_disposition) | |
filename = params.get('filename') | |
if filename: | |
# We need to sanitize the filename to prevent directory traversal | |
# in case the filename contains ".." path parts. | |
filename = sanitize_content_filename(filename) | |
return filename or default_filename | |
def _get_http_response_filename(resp, link): | |
# type: (Response, Link) -> str | |
"""Get an ideal filename from the given HTTP response, falling back to | |
the link filename if not provided. | |
""" | |
filename = link.filename # fallback | |
# Have a look at the Content-Disposition header for a better guess | |
content_disposition = resp.headers.get('content-disposition') | |
if content_disposition: | |
filename = parse_content_disposition(content_disposition, filename) | |
ext = splitext(filename)[1] # type: Optional[str] | |
if not ext: | |
ext = mimetypes.guess_extension( | |
resp.headers.get('content-type', '') | |
) | |
if ext: | |
filename += ext | |
if not ext and link.url != resp.url: | |
ext = os.path.splitext(resp.url)[1] | |
if ext: | |
filename += ext | |
return filename | |
def _http_get_download(session, link): | |
# type: (PipSession, Link) -> Response | |
target_url = link.url.split('#', 1)[0] | |
resp = session.get( | |
target_url, | |
# We use Accept-Encoding: identity here because requests | |
# defaults to accepting compressed responses. This breaks in | |
# a variety of ways depending on how the server is configured. | |
# - Some servers will notice that the file isn't a compressible | |
# file and will leave the file alone and with an empty | |
# Content-Encoding | |
# - Some servers will notice that the file is already | |
# compressed and will leave the file alone and will add a | |
# Content-Encoding: gzip header | |
# - Some servers won't notice anything at all and will take | |
# a file that's already been compressed and compress it again | |
# and set the Content-Encoding: gzip header | |
# By setting this to request only the identity encoding We're | |
# hoping to eliminate the third case. Hopefully there does not | |
# exist a server which when given a file will notice it is | |
# already compressed and that you're not asking for a | |
# compressed file and will then decompress it before sending | |
# because if that's the case I don't think it'll ever be | |
# possible to make this work. | |
headers={"Accept-Encoding": "identity"}, | |
stream=True, | |
) | |
resp.raise_for_status() | |
return resp | |
class Download(object): | |
def __init__( | |
self, | |
response, # type: Response | |
filename, # type: str | |
chunks, # type: Iterable[bytes] | |
): | |
# type: (...) -> None | |
self.response = response | |
self.filename = filename | |
self.chunks = chunks | |
class Downloader(object): | |
def __init__( | |
self, | |
session, # type: PipSession | |
progress_bar, # type: str | |
): | |
# type: (...) -> None | |
self._session = session | |
self._progress_bar = progress_bar | |
def __call__(self, link): | |
# type: (Link) -> Download | |
try: | |
resp = _http_get_download(self._session, link) | |
except requests.HTTPError as e: | |
logger.critical( | |
"HTTP error %s while getting %s", e.response.status_code, link | |
) | |
raise | |
return Download( | |
resp, | |
_get_http_response_filename(resp, link), | |
_prepare_download(resp, link, self._progress_bar), | |
) |
"""PipSession and supporting code, containing all pip-specific | |
network request configuration and behavior. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import email.utils | |
import json | |
import logging | |
import mimetypes | |
import os | |
import platform | |
import sys | |
import warnings | |
from pip._vendor import requests, six, urllib3 | |
from pip._vendor.cachecontrol import CacheControlAdapter | |
from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter | |
from pip._vendor.requests.models import Response | |
from pip._vendor.requests.structures import CaseInsensitiveDict | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._vendor.urllib3.exceptions import InsecureRequestWarning | |
from pip import __version__ | |
from pip._internal.network.auth import MultiDomainBasicAuth | |
from pip._internal.network.cache import SafeFileCache | |
# Import ssl from compat so the initial import occurs in only one place. | |
from pip._internal.utils.compat import has_tls, ipaddress | |
from pip._internal.utils.glibc import libc_ver | |
from pip._internal.utils.misc import ( | |
build_url_from_netloc, | |
get_installed_version, | |
parse_netloc, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import url_to_path | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Iterator, List, Optional, Tuple, Union, | |
) | |
from pip._internal.models.link import Link | |
SecureOrigin = Tuple[str, str, Optional[Union[int, str]]] | |
logger = logging.getLogger(__name__) | |
# Ignore warning raised when using --trusted-host. | |
warnings.filterwarnings("ignore", category=InsecureRequestWarning) | |
SECURE_ORIGINS = [ | |
# protocol, hostname, port | |
# Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC) | |
("https", "*", "*"), | |
("*", "localhost", "*"), | |
("*", "127.0.0.0/8", "*"), | |
("*", "::1/128", "*"), | |
("file", "*", None), | |
# ssh is always secure. | |
("ssh", "*", "*"), | |
] # type: List[SecureOrigin] | |
# These are environment variables present when running under various | |
# CI systems. For each variable, some CI systems that use the variable | |
# are indicated. The collection was chosen so that for each of a number | |
# of popular systems, at least one of the environment variables is used. | |
# This list is used to provide some indication of and lower bound for | |
# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive. | |
# For more background, see: https://github.com/pypa/pip/issues/5499 | |
CI_ENVIRONMENT_VARIABLES = ( | |
# Azure Pipelines | |
'BUILD_BUILDID', | |
# Jenkins | |
'BUILD_ID', | |
# AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI | |
'CI', | |
# Explicit environment variable. | |
'PIP_IS_CI', | |
) | |
def looks_like_ci(): | |
# type: () -> bool | |
""" | |
Return whether it looks like pip is running under CI. | |
""" | |
# We don't use the method of checking for a tty (e.g. using isatty()) | |
# because some CI systems mimic a tty (e.g. Travis CI). Thus that | |
# method doesn't provide definitive information in either direction. | |
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES) | |
def user_agent(): | |
""" | |
Return a string representing the user agent. | |
""" | |
data = { | |
"installer": {"name": "pip", "version": __version__}, | |
"python": platform.python_version(), | |
"implementation": { | |
"name": platform.python_implementation(), | |
}, | |
} | |
if data["implementation"]["name"] == 'CPython': | |
data["implementation"]["version"] = platform.python_version() | |
elif data["implementation"]["name"] == 'PyPy': | |
if sys.pypy_version_info.releaselevel == 'final': | |
pypy_version_info = sys.pypy_version_info[:3] | |
else: | |
pypy_version_info = sys.pypy_version_info | |
data["implementation"]["version"] = ".".join( | |
[str(x) for x in pypy_version_info] | |
) | |
elif data["implementation"]["name"] == 'Jython': | |
# Complete Guess | |
data["implementation"]["version"] = platform.python_version() | |
elif data["implementation"]["name"] == 'IronPython': | |
# Complete Guess | |
data["implementation"]["version"] = platform.python_version() | |
if sys.platform.startswith("linux"): | |
from pip._vendor import distro | |
distro_infos = dict(filter( | |
lambda x: x[1], | |
zip(["name", "version", "id"], distro.linux_distribution()), | |
)) | |
libc = dict(filter( | |
lambda x: x[1], | |
zip(["lib", "version"], libc_ver()), | |
)) | |
if libc: | |
distro_infos["libc"] = libc | |
if distro_infos: | |
data["distro"] = distro_infos | |
if sys.platform.startswith("darwin") and platform.mac_ver()[0]: | |
data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} | |
if platform.system(): | |
data.setdefault("system", {})["name"] = platform.system() | |
if platform.release(): | |
data.setdefault("system", {})["release"] = platform.release() | |
if platform.machine(): | |
data["cpu"] = platform.machine() | |
if has_tls(): | |
import _ssl as ssl | |
data["openssl_version"] = ssl.OPENSSL_VERSION | |
setuptools_version = get_installed_version("setuptools") | |
if setuptools_version is not None: | |
data["setuptools_version"] = setuptools_version | |
# Use None rather than False so as not to give the impression that | |
# pip knows it is not being run under CI. Rather, it is a null or | |
# inconclusive result. Also, we include some value rather than no | |
# value to make it easier to know that the check has been run. | |
data["ci"] = True if looks_like_ci() else None | |
user_data = os.environ.get("PIP_USER_AGENT_USER_DATA") | |
if user_data is not None: | |
data["user_data"] = user_data | |
return "{data[installer][name]}/{data[installer][version]} {json}".format( | |
data=data, | |
json=json.dumps(data, separators=(",", ":"), sort_keys=True), | |
) | |
class LocalFSAdapter(BaseAdapter): | |
def send(self, request, stream=None, timeout=None, verify=None, cert=None, | |
proxies=None): | |
pathname = url_to_path(request.url) | |
resp = Response() | |
resp.status_code = 200 | |
resp.url = request.url | |
try: | |
stats = os.stat(pathname) | |
except OSError as exc: | |
resp.status_code = 404 | |
resp.raw = exc | |
else: | |
modified = email.utils.formatdate(stats.st_mtime, usegmt=True) | |
content_type = mimetypes.guess_type(pathname)[0] or "text/plain" | |
resp.headers = CaseInsensitiveDict({ | |
"Content-Type": content_type, | |
"Content-Length": stats.st_size, | |
"Last-Modified": modified, | |
}) | |
resp.raw = open(pathname, "rb") | |
resp.close = resp.raw.close | |
return resp | |
def close(self): | |
pass | |
class InsecureHTTPAdapter(HTTPAdapter): | |
def cert_verify(self, conn, url, verify, cert): | |
super(InsecureHTTPAdapter, self).cert_verify( | |
conn=conn, url=url, verify=False, cert=cert | |
) | |
class PipSession(requests.Session): | |
timeout = None # type: Optional[int] | |
def __init__(self, *args, **kwargs): | |
""" | |
:param trusted_hosts: Domains not to emit warnings for when not using | |
HTTPS. | |
""" | |
retries = kwargs.pop("retries", 0) | |
cache = kwargs.pop("cache", None) | |
trusted_hosts = kwargs.pop("trusted_hosts", []) # type: List[str] | |
index_urls = kwargs.pop("index_urls", None) | |
super(PipSession, self).__init__(*args, **kwargs) | |
# Namespace the attribute with "pip_" just in case to prevent | |
# possible conflicts with the base class. | |
self.pip_trusted_origins = [] # type: List[Tuple[str, Optional[int]]] | |
# Attach our User Agent to the request | |
self.headers["User-Agent"] = user_agent() | |
# Attach our Authentication handler to the session | |
self.auth = MultiDomainBasicAuth(index_urls=index_urls) | |
# Create our urllib3.Retry instance which will allow us to customize | |
# how we handle retries. | |
retries = urllib3.Retry( | |
# Set the total number of retries that a particular request can | |
# have. | |
total=retries, | |
# A 503 error from PyPI typically means that the Fastly -> Origin | |
# connection got interrupted in some way. A 503 error in general | |
# is typically considered a transient error so we'll go ahead and | |
# retry it. | |
# A 500 may indicate transient error in Amazon S3 | |
# A 520 or 527 - may indicate transient error in CloudFlare | |
status_forcelist=[500, 503, 520, 527], | |
# Add a small amount of back off between failed requests in | |
# order to prevent hammering the service. | |
backoff_factor=0.25, | |
) | |
# We want to _only_ cache responses on securely fetched origins. We do | |
# this because we can't validate the response of an insecurely fetched | |
# origin, and we don't want someone to be able to poison the cache and | |
# require manual eviction from the cache to fix it. | |
if cache: | |
secure_adapter = CacheControlAdapter( | |
cache=SafeFileCache(cache), | |
max_retries=retries, | |
) | |
else: | |
secure_adapter = HTTPAdapter(max_retries=retries) | |
# Our Insecure HTTPAdapter disables HTTPS validation. It does not | |
# support caching (see above) so we'll use it for all http:// URLs as | |
# well as any https:// host that we've marked as ignoring TLS errors | |
# for. | |
insecure_adapter = InsecureHTTPAdapter(max_retries=retries) | |
# Save this for later use in add_insecure_host(). | |
self._insecure_adapter = insecure_adapter | |
self.mount("https://", secure_adapter) | |
self.mount("http://", insecure_adapter) | |
# Enable file:// urls | |
self.mount("file://", LocalFSAdapter()) | |
for host in trusted_hosts: | |
self.add_trusted_host(host, suppress_logging=True) | |
def add_trusted_host(self, host, source=None, suppress_logging=False): | |
# type: (str, Optional[str], bool) -> None | |
""" | |
:param host: It is okay to provide a host that has previously been | |
added. | |
:param source: An optional source string, for logging where the host | |
string came from. | |
""" | |
if not suppress_logging: | |
msg = 'adding trusted host: {!r}'.format(host) | |
if source is not None: | |
msg += ' (from {})'.format(source) | |
logger.info(msg) | |
host_port = parse_netloc(host) | |
if host_port not in self.pip_trusted_origins: | |
self.pip_trusted_origins.append(host_port) | |
self.mount(build_url_from_netloc(host) + '/', self._insecure_adapter) | |
if not host_port[1]: | |
# Mount wildcard ports for the same host. | |
self.mount( | |
build_url_from_netloc(host) + ':', | |
self._insecure_adapter | |
) | |
def iter_secure_origins(self): | |
# type: () -> Iterator[SecureOrigin] | |
for secure_origin in SECURE_ORIGINS: | |
yield secure_origin | |
for host, port in self.pip_trusted_origins: | |
yield ('*', host, '*' if port is None else port) | |
def is_secure_origin(self, location): | |
# type: (Link) -> bool | |
# Determine if this url used a secure transport mechanism | |
parsed = urllib_parse.urlparse(str(location)) | |
origin_protocol, origin_host, origin_port = ( | |
parsed.scheme, parsed.hostname, parsed.port, | |
) | |
# The protocol to use to see if the protocol matches. | |
# Don't count the repository type as part of the protocol: in | |
# cases such as "git+ssh", only use "ssh". (I.e., Only verify against | |
# the last scheme.) | |
origin_protocol = origin_protocol.rsplit('+', 1)[-1] | |
# Determine if our origin is a secure origin by looking through our | |
# hardcoded list of secure origins, as well as any additional ones | |
# configured on this PackageFinder instance. | |
for secure_origin in self.iter_secure_origins(): | |
secure_protocol, secure_host, secure_port = secure_origin | |
if origin_protocol != secure_protocol and secure_protocol != "*": | |
continue | |
try: | |
addr = ipaddress.ip_address( | |
None | |
if origin_host is None | |
else six.ensure_text(origin_host) | |
) | |
network = ipaddress.ip_network( | |
six.ensure_text(secure_host) | |
) | |
except ValueError: | |
# We don't have both a valid address or a valid network, so | |
# we'll check this origin against hostnames. | |
if ( | |
origin_host and | |
origin_host.lower() != secure_host.lower() and | |
secure_host != "*" | |
): | |
continue | |
else: | |
# We have a valid address and network, so see if the address | |
# is contained within the network. | |
if addr not in network: | |
continue | |
# Check to see if the port matches. | |
if ( | |
origin_port != secure_port and | |
secure_port != "*" and | |
secure_port is not None | |
): | |
continue | |
# If we've gotten here, then this origin matches the current | |
# secure origin and we should return True | |
return True | |
# If we've gotten to this point, then the origin isn't secure and we | |
# will not accept it as a valid location to search. We will however | |
# log a warning that we are ignoring it. | |
logger.warning( | |
"The repository located at %s is not a trusted or secure host and " | |
"is being ignored. If this repository is available via HTTPS we " | |
"recommend you use HTTPS instead, otherwise you may silence " | |
"this warning and allow it anyway with '--trusted-host %s'.", | |
origin_host, | |
origin_host, | |
) | |
return False | |
def request(self, method, url, *args, **kwargs): | |
# Allow setting a default timeout on a session | |
kwargs.setdefault("timeout", self.timeout) | |
# Dispatch the actual request | |
return super(PipSession, self).request(method, url, *args, **kwargs) |
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Iterator | |
def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE): | |
# type: (Response, int) -> Iterator[bytes] | |
"""Given a requests Response, provide the data chunks. | |
""" | |
try: | |
# Special case for urllib3. | |
for chunk in response.raw.stream( | |
chunk_size, | |
# We use decode_content=False here because we don't | |
# want urllib3 to mess with the raw bytes we get | |
# from the server. If we decompress inside of | |
# urllib3 then we cannot verify the checksum | |
# because the checksum will be of the compressed | |
# file. This breakage will only occur if the | |
# server adds a Content-Encoding header, which | |
# depends on how the server was configured: | |
# - Some servers will notice that the file isn't a | |
# compressible file and will leave the file alone | |
# and with an empty Content-Encoding | |
# - Some servers will notice that the file is | |
# already compressed and will leave the file | |
# alone and will add a Content-Encoding: gzip | |
# header | |
# - Some servers won't notice anything at all and | |
# will take a file that's already been compressed | |
# and compress it again and set the | |
# Content-Encoding: gzip header | |
# | |
# By setting this not to decode automatically we | |
# hope to eliminate problems with the second case. | |
decode_content=False, | |
): | |
yield chunk | |
except AttributeError: | |
# Standard file-like object. | |
while True: | |
chunk = response.raw.read(chunk_size) | |
if not chunk: | |
break | |
yield chunk |
"""xmlrpclib.Transport implementation | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import logging | |
from pip._vendor import requests | |
# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is | |
# why we ignore the type on this import | |
from pip._vendor.six.moves import xmlrpc_client # type: ignore | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
logger = logging.getLogger(__name__) | |
class PipXmlrpcTransport(xmlrpc_client.Transport): | |
"""Provide a `xmlrpclib.Transport` implementation via a `PipSession` | |
object. | |
""" | |
def __init__(self, index_url, session, use_datetime=False): | |
xmlrpc_client.Transport.__init__(self, use_datetime) | |
index_parts = urllib_parse.urlparse(index_url) | |
self._scheme = index_parts.scheme | |
self._session = session | |
def request(self, host, handler, request_body, verbose=False): | |
parts = (self._scheme, host, handler, None, None, None) | |
url = urllib_parse.urlunparse(parts) | |
try: | |
headers = {'Content-Type': 'text/xml'} | |
response = self._session.post(url, data=request_body, | |
headers=headers, stream=True) | |
response.raise_for_status() | |
self.verbose = verbose | |
return self.parse_response(response.raw) | |
except requests.HTTPError as exc: | |
logger.critical( | |
"HTTP error %s while getting %s", | |
exc.response.status_code, url, | |
) | |
raise |
"""Metadata generation logic for source distributions. | |
""" | |
import logging | |
import os | |
from pip._internal.utils.subprocess import runner_with_spinner_message | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from pip._internal.build_env import BuildEnvironment | |
from pip._vendor.pep517.wrappers import Pep517HookCaller | |
logger = logging.getLogger(__name__) | |
def generate_metadata(build_env, backend): | |
# type: (BuildEnvironment, Pep517HookCaller) -> str | |
"""Generate metadata using mechanisms described in PEP 517. | |
Returns the generated metadata directory. | |
""" | |
metadata_tmpdir = TempDirectory( | |
kind="modern-metadata", globally_managed=True | |
) | |
metadata_dir = metadata_tmpdir.path | |
with build_env: | |
# Note that Pep517HookCaller implements a fallback for | |
# prepare_metadata_for_build_wheel, so we don't have to | |
# consider the possibility that this hook doesn't exist. | |
runner = runner_with_spinner_message("Preparing wheel metadata") | |
with backend.subprocess_runner(runner): | |
distinfo_dir = backend.prepare_metadata_for_build_wheel( | |
metadata_dir | |
) | |
return os.path.join(metadata_dir, distinfo_dir) |
"""Metadata generation logic for legacy source distributions. | |
""" | |
import logging | |
import os | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.utils.misc import ensure_dir | |
from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args | |
from pip._internal.utils.subprocess import call_subprocess | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.vcs import vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional | |
from pip._internal.build_env import BuildEnvironment | |
logger = logging.getLogger(__name__) | |
def _find_egg_info(source_directory, is_editable): | |
# type: (str, bool) -> str | |
"""Find an .egg-info in `source_directory`, based on `is_editable`. | |
""" | |
def looks_like_virtual_env(path): | |
# type: (str) -> bool | |
return ( | |
os.path.lexists(os.path.join(path, 'bin', 'python')) or | |
os.path.exists(os.path.join(path, 'Scripts', 'Python.exe')) | |
) | |
def locate_editable_egg_info(base): | |
# type: (str) -> List[str] | |
candidates = [] # type: List[str] | |
for root, dirs, files in os.walk(base): | |
for dir_ in vcs.dirnames: | |
if dir_ in dirs: | |
dirs.remove(dir_) | |
# Iterate over a copy of ``dirs``, since mutating | |
# a list while iterating over it can cause trouble. | |
# (See https://github.com/pypa/pip/pull/462.) | |
for dir_ in list(dirs): | |
if looks_like_virtual_env(os.path.join(root, dir_)): | |
dirs.remove(dir_) | |
# Also don't search through tests | |
elif dir_ == 'test' or dir_ == 'tests': | |
dirs.remove(dir_) | |
candidates.extend(os.path.join(root, dir_) for dir_ in dirs) | |
return [f for f in candidates if f.endswith('.egg-info')] | |
def depth_of_directory(dir_): | |
# type: (str) -> int | |
return ( | |
dir_.count(os.path.sep) + | |
(os.path.altsep and dir_.count(os.path.altsep) or 0) | |
) | |
base = source_directory | |
if is_editable: | |
filenames = locate_editable_egg_info(base) | |
else: | |
base = os.path.join(base, 'pip-egg-info') | |
filenames = os.listdir(base) | |
if not filenames: | |
raise InstallationError( | |
"Files/directories not found in {}".format(base) | |
) | |
# If we have more than one match, we pick the toplevel one. This | |
# can easily be the case if there is a dist folder which contains | |
# an extracted tarball for testing purposes. | |
if len(filenames) > 1: | |
filenames.sort(key=depth_of_directory) | |
return os.path.join(base, filenames[0]) | |
def generate_metadata( | |
build_env, # type: BuildEnvironment | |
setup_py_path, # type: str | |
source_dir, # type: str | |
editable, # type: bool | |
isolated, # type: bool | |
details, # type: str | |
): | |
# type: (...) -> str | |
"""Generate metadata using setup.py-based defacto mechanisms. | |
Returns the generated metadata directory. | |
""" | |
logger.debug( | |
'Running setup.py (path:%s) egg_info for package %s', | |
setup_py_path, details, | |
) | |
egg_info_dir = None # type: Optional[str] | |
# For non-editable installs, don't put the .egg-info files at the root, | |
# to avoid confusion due to the source code being considered an installed | |
# egg. | |
if not editable: | |
egg_info_dir = os.path.join(source_dir, 'pip-egg-info') | |
# setuptools complains if the target directory does not exist. | |
ensure_dir(egg_info_dir) | |
args = make_setuptools_egg_info_args( | |
setup_py_path, | |
egg_info_dir=egg_info_dir, | |
no_user_config=isolated, | |
) | |
with build_env: | |
call_subprocess( | |
args, | |
cwd=source_dir, | |
command_desc='python setup.py egg_info', | |
) | |
# Return the .egg-info directory. | |
return _find_egg_info(source_dir, editable) |
import logging | |
import os | |
from pip._internal.utils.subprocess import runner_with_spinner_message | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional | |
from pip._vendor.pep517.wrappers import Pep517HookCaller | |
logger = logging.getLogger(__name__) | |
def build_wheel_pep517( | |
name, # type: str | |
backend, # type: Pep517HookCaller | |
metadata_directory, # type: str | |
build_options, # type: List[str] | |
tempd, # type: str | |
): | |
# type: (...) -> Optional[str] | |
"""Build one InstallRequirement using the PEP 517 build process. | |
Returns path to wheel if successfully built. Otherwise, returns None. | |
""" | |
assert metadata_directory is not None | |
if build_options: | |
# PEP 517 does not support --build-options | |
logger.error('Cannot build wheel for %s using PEP 517 when ' | |
'--build-option is present' % (name,)) | |
return None | |
try: | |
logger.debug('Destination directory: %s', tempd) | |
runner = runner_with_spinner_message( | |
'Building wheel for {} (PEP 517)'.format(name) | |
) | |
with backend.subprocess_runner(runner): | |
wheel_name = backend.build_wheel( | |
tempd, | |
metadata_directory=metadata_directory, | |
) | |
except Exception: | |
logger.error('Failed building wheel for %s', name) | |
return None | |
return os.path.join(tempd, wheel_name) |
import logging | |
import os.path | |
from pip._internal.utils.setuptools_build import ( | |
make_setuptools_bdist_wheel_args, | |
) | |
from pip._internal.utils.subprocess import ( | |
LOG_DIVIDER, | |
call_subprocess, | |
format_command_args, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.ui import open_spinner | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional, Text | |
logger = logging.getLogger(__name__) | |
def format_command_result( | |
command_args, # type: List[str] | |
command_output, # type: Text | |
): | |
# type: (...) -> str | |
"""Format command information for logging.""" | |
command_desc = format_command_args(command_args) | |
text = 'Command arguments: {}\n'.format(command_desc) | |
if not command_output: | |
text += 'Command output: None' | |
elif logger.getEffectiveLevel() > logging.DEBUG: | |
text += 'Command output: [use --verbose to show]' | |
else: | |
if not command_output.endswith('\n'): | |
command_output += '\n' | |
text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER) | |
return text | |
def get_legacy_build_wheel_path( | |
names, # type: List[str] | |
temp_dir, # type: str | |
name, # type: str | |
command_args, # type: List[str] | |
command_output, # type: Text | |
): | |
# type: (...) -> Optional[str] | |
"""Return the path to the wheel in the temporary build directory.""" | |
# Sort for determinism. | |
names = sorted(names) | |
if not names: | |
msg = ( | |
'Legacy build of wheel for {!r} created no files.\n' | |
).format(name) | |
msg += format_command_result(command_args, command_output) | |
logger.warning(msg) | |
return None | |
if len(names) > 1: | |
msg = ( | |
'Legacy build of wheel for {!r} created more than one file.\n' | |
'Filenames (choosing first): {}\n' | |
).format(name, names) | |
msg += format_command_result(command_args, command_output) | |
logger.warning(msg) | |
return os.path.join(temp_dir, names[0]) | |
def build_wheel_legacy( | |
name, # type: str | |
setup_py_path, # type: str | |
source_dir, # type: str | |
global_options, # type: List[str] | |
build_options, # type: List[str] | |
tempd, # type: str | |
): | |
# type: (...) -> Optional[str] | |
"""Build one unpacked package using the "legacy" build process. | |
Returns path to wheel if successfully built. Otherwise, returns None. | |
""" | |
wheel_args = make_setuptools_bdist_wheel_args( | |
setup_py_path, | |
global_options=global_options, | |
build_options=build_options, | |
destination_dir=tempd, | |
) | |
spin_message = 'Building wheel for %s (setup.py)' % (name,) | |
with open_spinner(spin_message) as spinner: | |
logger.debug('Destination directory: %s', tempd) | |
try: | |
output = call_subprocess( | |
wheel_args, | |
cwd=source_dir, | |
spinner=spinner, | |
) | |
except Exception: | |
spinner.finish("error") | |
logger.error('Failed building wheel for %s', name) | |
return None | |
names = os.listdir(tempd) | |
wheel_path = get_legacy_build_wheel_path( | |
names=names, | |
temp_dir=tempd, | |
name=name, | |
command_args=wheel_args, | |
command_output=output, | |
) | |
return wheel_path |
"""Validation of dependencies of packages | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
import logging | |
from collections import namedtuple | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.pkg_resources import RequirementParseError | |
from pip._internal.distributions import ( | |
make_distribution_for_install_requirement, | |
) | |
from pip._internal.utils.misc import get_installed_distributions | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
logger = logging.getLogger(__name__) | |
if MYPY_CHECK_RUNNING: | |
from pip._internal.req.req_install import InstallRequirement | |
from typing import ( | |
Any, Callable, Dict, Optional, Set, Tuple, List | |
) | |
# Shorthands | |
PackageSet = Dict[str, 'PackageDetails'] | |
Missing = Tuple[str, Any] | |
Conflicting = Tuple[str, str, Any] | |
MissingDict = Dict[str, List[Missing]] | |
ConflictingDict = Dict[str, List[Conflicting]] | |
CheckResult = Tuple[MissingDict, ConflictingDict] | |
PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) | |
def create_package_set_from_installed(**kwargs): | |
# type: (**Any) -> Tuple[PackageSet, bool] | |
"""Converts a list of distributions into a PackageSet. | |
""" | |
# Default to using all packages installed on the system | |
if kwargs == {}: | |
kwargs = {"local_only": False, "skip": ()} | |
package_set = {} | |
problems = False | |
for dist in get_installed_distributions(**kwargs): | |
name = canonicalize_name(dist.project_name) | |
try: | |
package_set[name] = PackageDetails(dist.version, dist.requires()) | |
except RequirementParseError as e: | |
# Don't crash on broken metadata | |
logger.warning("Error parsing requirements for %s: %s", name, e) | |
problems = True | |
return package_set, problems | |
def check_package_set(package_set, should_ignore=None): | |
# type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult | |
"""Check if a package set is consistent | |
If should_ignore is passed, it should be a callable that takes a | |
package name and returns a boolean. | |
""" | |
if should_ignore is None: | |
def should_ignore(name): | |
return False | |
missing = {} | |
conflicting = {} | |
for package_name in package_set: | |
# Info about dependencies of package_name | |
missing_deps = set() # type: Set[Missing] | |
conflicting_deps = set() # type: Set[Conflicting] | |
if should_ignore(package_name): | |
continue | |
for req in package_set[package_name].requires: | |
name = canonicalize_name(req.project_name) # type: str | |
# Check if it's missing | |
if name not in package_set: | |
missed = True | |
if req.marker is not None: | |
missed = req.marker.evaluate() | |
if missed: | |
missing_deps.add((name, req)) | |
continue | |
# Check if there's a conflict | |
version = package_set[name].version # type: str | |
if not req.specifier.contains(version, prereleases=True): | |
conflicting_deps.add((name, version, req)) | |
if missing_deps: | |
missing[package_name] = sorted(missing_deps, key=str) | |
if conflicting_deps: | |
conflicting[package_name] = sorted(conflicting_deps, key=str) | |
return missing, conflicting | |
def check_install_conflicts(to_install): | |
# type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult] | |
"""For checking if the dependency graph would be consistent after \ | |
installing given requirements | |
""" | |
# Start from the current state | |
package_set, _ = create_package_set_from_installed() | |
# Install packages | |
would_be_installed = _simulate_installation_of(to_install, package_set) | |
# Only warn about directly-dependent packages; create a whitelist of them | |
whitelist = _create_whitelist(would_be_installed, package_set) | |
return ( | |
package_set, | |
check_package_set( | |
package_set, should_ignore=lambda name: name not in whitelist | |
) | |
) | |
def _simulate_installation_of(to_install, package_set): | |
# type: (List[InstallRequirement], PackageSet) -> Set[str] | |
"""Computes the version of packages after installing to_install. | |
""" | |
# Keep track of packages that were installed | |
installed = set() | |
# Modify it as installing requirement_set would (assuming no errors) | |
for inst_req in to_install: | |
abstract_dist = make_distribution_for_install_requirement(inst_req) | |
dist = abstract_dist.get_pkg_resources_distribution() | |
name = canonicalize_name(dist.key) | |
package_set[name] = PackageDetails(dist.version, dist.requires()) | |
installed.add(name) | |
return installed | |
def _create_whitelist(would_be_installed, package_set): | |
# type: (Set[str], PackageSet) -> Set[str] | |
packages_affected = set(would_be_installed) | |
for package_name in package_set: | |
if package_name in packages_affected: | |
continue | |
for req in package_set[package_name].requires: | |
if canonicalize_name(req.name) in packages_affected: | |
packages_affected.add(package_name) | |
break | |
return packages_affected |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import collections | |
import logging | |
import os | |
import re | |
from pip._vendor import six | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.pkg_resources import RequirementParseError | |
from pip._internal.exceptions import BadCommand, InstallationError | |
from pip._internal.req.constructors import ( | |
install_req_from_editable, | |
install_req_from_line, | |
) | |
from pip._internal.req.req_file import COMMENT_RE | |
from pip._internal.utils.misc import ( | |
dist_is_editable, | |
get_installed_distributions, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union | |
) | |
from pip._internal.cache import WheelCache | |
from pip._vendor.pkg_resources import ( | |
Distribution, Requirement | |
) | |
RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] | |
logger = logging.getLogger(__name__) | |
def freeze( | |
requirement=None, # type: Optional[List[str]] | |
find_links=None, # type: Optional[List[str]] | |
local_only=None, # type: Optional[bool] | |
user_only=None, # type: Optional[bool] | |
paths=None, # type: Optional[List[str]] | |
skip_regex=None, # type: Optional[str] | |
isolated=False, # type: bool | |
wheel_cache=None, # type: Optional[WheelCache] | |
exclude_editable=False, # type: bool | |
skip=() # type: Container[str] | |
): | |
# type: (...) -> Iterator[str] | |
find_links = find_links or [] | |
skip_match = None | |
if skip_regex: | |
skip_match = re.compile(skip_regex).search | |
for link in find_links: | |
yield '-f %s' % link | |
installations = {} # type: Dict[str, FrozenRequirement] | |
for dist in get_installed_distributions(local_only=local_only, | |
skip=(), | |
user_only=user_only, | |
paths=paths): | |
try: | |
req = FrozenRequirement.from_dist(dist) | |
except RequirementParseError as exc: | |
# We include dist rather than dist.project_name because the | |
# dist string includes more information, like the version and | |
# location. We also include the exception message to aid | |
# troubleshooting. | |
logger.warning( | |
'Could not generate requirement for distribution %r: %s', | |
dist, exc | |
) | |
continue | |
if exclude_editable and req.editable: | |
continue | |
installations[req.canonical_name] = req | |
if requirement: | |
# the options that don't get turned into an InstallRequirement | |
# should only be emitted once, even if the same option is in multiple | |
# requirements files, so we need to keep track of what has been emitted | |
# so that we don't emit it again if it's seen again | |
emitted_options = set() # type: Set[str] | |
# keep track of which files a requirement is in so that we can | |
# give an accurate warning if a requirement appears multiple times. | |
req_files = collections.defaultdict(list) # type: Dict[str, List[str]] | |
for req_file_path in requirement: | |
with open(req_file_path) as req_file: | |
for line in req_file: | |
if (not line.strip() or | |
line.strip().startswith('#') or | |
(skip_match and skip_match(line)) or | |
line.startswith(( | |
'-r', '--requirement', | |
'-Z', '--always-unzip', | |
'-f', '--find-links', | |
'-i', '--index-url', | |
'--pre', | |
'--trusted-host', | |
'--process-dependency-links', | |
'--extra-index-url'))): | |
line = line.rstrip() | |
if line not in emitted_options: | |
emitted_options.add(line) | |
yield line | |
continue | |
if line.startswith('-e') or line.startswith('--editable'): | |
if line.startswith('-e'): | |
line = line[2:].strip() | |
else: | |
line = line[len('--editable'):].strip().lstrip('=') | |
line_req = install_req_from_editable( | |
line, | |
isolated=isolated, | |
wheel_cache=wheel_cache, | |
) | |
else: | |
line_req = install_req_from_line( | |
COMMENT_RE.sub('', line).strip(), | |
isolated=isolated, | |
wheel_cache=wheel_cache, | |
) | |
if not line_req.name: | |
logger.info( | |
"Skipping line in requirement file [%s] because " | |
"it's not clear what it would install: %s", | |
req_file_path, line.strip(), | |
) | |
logger.info( | |
" (add #egg=PackageName to the URL to avoid" | |
" this warning)" | |
) | |
else: | |
line_req_canonical_name = canonicalize_name( | |
line_req.name) | |
if line_req_canonical_name not in installations: | |
# either it's not installed, or it is installed | |
# but has been processed already | |
if not req_files[line_req.name]: | |
logger.warning( | |
"Requirement file [%s] contains %s, but " | |
"package %r is not installed", | |
req_file_path, | |
COMMENT_RE.sub('', line).strip(), | |
line_req.name | |
) | |
else: | |
req_files[line_req.name].append(req_file_path) | |
else: | |
yield str(installations[ | |
line_req_canonical_name]).rstrip() | |
del installations[line_req_canonical_name] | |
req_files[line_req.name].append(req_file_path) | |
# Warn about requirements that were included multiple times (in a | |
# single requirements file or in different requirements files). | |
for name, files in six.iteritems(req_files): | |
if len(files) > 1: | |
logger.warning("Requirement %s included multiple times [%s]", | |
name, ', '.join(sorted(set(files)))) | |
yield( | |
'## The following requirements were added by ' | |
'pip freeze:' | |
) | |
for installation in sorted( | |
installations.values(), key=lambda x: x.name.lower()): | |
if installation.canonical_name not in skip: | |
yield str(installation).rstrip() | |
def get_requirement_info(dist): | |
# type: (Distribution) -> RequirementInfo | |
""" | |
Compute and return values (req, editable, comments) for use in | |
FrozenRequirement.from_dist(). | |
""" | |
if not dist_is_editable(dist): | |
return (None, False, []) | |
location = os.path.normcase(os.path.abspath(dist.location)) | |
from pip._internal.vcs import vcs, RemoteNotFoundError | |
vcs_backend = vcs.get_backend_for_dir(location) | |
if vcs_backend is None: | |
req = dist.as_requirement() | |
logger.debug( | |
'No VCS found for editable requirement "%s" in: %r', req, | |
location, | |
) | |
comments = [ | |
'# Editable install with no version control ({})'.format(req) | |
] | |
return (location, True, comments) | |
try: | |
req = vcs_backend.get_src_requirement(location, dist.project_name) | |
except RemoteNotFoundError: | |
req = dist.as_requirement() | |
comments = [ | |
'# Editable {} install with no remote ({})'.format( | |
type(vcs_backend).__name__, req, | |
) | |
] | |
return (location, True, comments) | |
except BadCommand: | |
logger.warning( | |
'cannot determine version of editable source in %s ' | |
'(%s command not found in path)', | |
location, | |
vcs_backend.name, | |
) | |
return (None, True, []) | |
except InstallationError as exc: | |
logger.warning( | |
"Error when trying to get requirement for VCS system %s, " | |
"falling back to uneditable format", exc | |
) | |
else: | |
if req is not None: | |
return (req, True, []) | |
logger.warning( | |
'Could not determine repository location of %s', location | |
) | |
comments = ['## !! Could not determine repository location'] | |
return (None, False, comments) | |
class FrozenRequirement(object): | |
def __init__(self, name, req, editable, comments=()): | |
# type: (str, Union[str, Requirement], bool, Iterable[str]) -> None | |
self.name = name | |
self.canonical_name = canonicalize_name(name) | |
self.req = req | |
self.editable = editable | |
self.comments = comments | |
@classmethod | |
def from_dist(cls, dist): | |
# type: (Distribution) -> FrozenRequirement | |
req, editable, comments = get_requirement_info(dist) | |
if req is None: | |
req = dist.as_requirement() | |
return cls(dist.project_name, req, editable, comments=comments) | |
def __str__(self): | |
req = self.req | |
if self.editable: | |
req = '-e %s' % req | |
return '\n'.join(list(self.comments) + [str(req)]) + '\n' |
"""For modules related to installing packages. | |
""" |
"""Legacy editable installation process, i.e. `setup.py develop`. | |
""" | |
import logging | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.setuptools_build import make_setuptools_develop_args | |
from pip._internal.utils.subprocess import call_subprocess | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional, Sequence | |
from pip._internal.build_env import BuildEnvironment | |
logger = logging.getLogger(__name__) | |
def install_editable( | |
install_options, # type: List[str] | |
global_options, # type: Sequence[str] | |
prefix, # type: Optional[str] | |
home, # type: Optional[str] | |
use_user_site, # type: bool | |
name, # type: str | |
setup_py_path, # type: str | |
isolated, # type: bool | |
build_env, # type: BuildEnvironment | |
unpacked_source_directory, # type: str | |
): | |
# type: (...) -> None | |
"""Install a package in editable mode. Most arguments are pass-through | |
to setuptools. | |
""" | |
logger.info('Running setup.py develop for %s', name) | |
args = make_setuptools_develop_args( | |
setup_py_path, | |
global_options=global_options, | |
install_options=install_options, | |
no_user_config=isolated, | |
prefix=prefix, | |
home=home, | |
use_user_site=use_user_site, | |
) | |
with indent_log(): | |
with build_env: | |
call_subprocess( | |
args, | |
cwd=unpacked_source_directory, | |
) |
"""Legacy installation process, i.e. `setup.py install`. | |
""" | |
import logging | |
import os | |
from distutils.util import change_root | |
from pip._internal.utils.deprecation import deprecated | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import ensure_dir | |
from pip._internal.utils.setuptools_build import make_setuptools_install_args | |
from pip._internal.utils.subprocess import runner_with_spinner_message | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional, Sequence | |
from pip._internal.models.scheme import Scheme | |
from pip._internal.req.req_install import InstallRequirement | |
logger = logging.getLogger(__name__) | |
def install( | |
install_req, # type: InstallRequirement | |
install_options, # type: List[str] | |
global_options, # type: Sequence[str] | |
root, # type: Optional[str] | |
home, # type: Optional[str] | |
prefix, # type: Optional[str] | |
use_user_site, # type: bool | |
pycompile, # type: bool | |
scheme, # type: Scheme | |
): | |
# type: (...) -> None | |
# Extend the list of global and install options passed on to | |
# the setup.py call with the ones from the requirements file. | |
# Options specified in requirements file override those | |
# specified on the command line, since the last option given | |
# to setup.py is the one that is used. | |
global_options = list(global_options) + \ | |
install_req.options.get('global_options', []) | |
install_options = list(install_options) + \ | |
install_req.options.get('install_options', []) | |
header_dir = scheme.headers | |
with TempDirectory(kind="record") as temp_dir: | |
record_filename = os.path.join(temp_dir.path, 'install-record.txt') | |
install_args = make_setuptools_install_args( | |
install_req.setup_py_path, | |
global_options=global_options, | |
install_options=install_options, | |
record_filename=record_filename, | |
root=root, | |
prefix=prefix, | |
header_dir=header_dir, | |
home=home, | |
use_user_site=use_user_site, | |
no_user_config=install_req.isolated, | |
pycompile=pycompile, | |
) | |
runner = runner_with_spinner_message( | |
"Running setup.py install for {}".format(install_req.name) | |
) | |
with indent_log(), install_req.build_env: | |
runner( | |
cmd=install_args, | |
cwd=install_req.unpacked_source_directory, | |
) | |
if not os.path.exists(record_filename): | |
logger.debug('Record file %s not found', record_filename) | |
return | |
install_req.install_succeeded = True | |
# We intentionally do not use any encoding to read the file because | |
# setuptools writes the file using distutils.file_util.write_file, | |
# which does not specify an encoding. | |
with open(record_filename) as f: | |
record_lines = f.read().splitlines() | |
def prepend_root(path): | |
# type: (str) -> str | |
if root is None or not os.path.isabs(path): | |
return path | |
else: | |
return change_root(root, path) | |
for line in record_lines: | |
directory = os.path.dirname(line) | |
if directory.endswith('.egg-info'): | |
egg_info_dir = prepend_root(directory) | |
break | |
else: | |
deprecated( | |
reason=( | |
"{} did not indicate that it installed an " | |
".egg-info directory. Only setup.py projects " | |
"generating .egg-info directories are supported." | |
).format(install_req), | |
replacement=( | |
"for maintainers: updating the setup.py of {0}. " | |
"For users: contact the maintainers of {0} to let " | |
"them know to update their setup.py.".format( | |
install_req.name | |
) | |
), | |
gone_in="20.2", | |
issue=6998, | |
) | |
# FIXME: put the record somewhere | |
return | |
new_lines = [] | |
for line in record_lines: | |
filename = line.strip() | |
if os.path.isdir(filename): | |
filename += os.path.sep | |
new_lines.append( | |
os.path.relpath(prepend_root(filename), egg_info_dir) | |
) | |
new_lines.sort() | |
ensure_dir(egg_info_dir) | |
inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') | |
with open(inst_files_path, 'w') as f: | |
f.write('\n'.join(new_lines) + '\n') |
"""Support for installing and building the "wheel" binary package format. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import collections | |
import compileall | |
import csv | |
import logging | |
import os.path | |
import re | |
import shutil | |
import stat | |
import sys | |
import warnings | |
from base64 import urlsafe_b64encode | |
from zipfile import ZipFile | |
from pip._vendor import pkg_resources | |
from pip._vendor.distlib.scripts import ScriptMaker | |
from pip._vendor.distlib.util import get_export_entry | |
from pip._vendor.six import StringIO | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.locations import get_major_minor_version | |
from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.unpacking import unpack_file | |
from pip._internal.utils.wheel import parse_wheel | |
if MYPY_CHECK_RUNNING: | |
from email.message import Message | |
from typing import ( | |
Dict, List, Optional, Sequence, Tuple, IO, Text, Any, | |
Iterable, Callable, Set, | |
) | |
from pip._internal.models.scheme import Scheme | |
InstalledCSVRow = Tuple[str, ...] | |
logger = logging.getLogger(__name__) | |
def normpath(src, p): | |
# type: (str, str) -> str | |
return os.path.relpath(src, p).replace(os.path.sep, '/') | |
def rehash(path, blocksize=1 << 20): | |
# type: (str, int) -> Tuple[str, str] | |
"""Return (encoded_digest, length) for path using hashlib.sha256()""" | |
h, length = hash_file(path, blocksize) | |
digest = 'sha256=' + urlsafe_b64encode( | |
h.digest() | |
).decode('latin1').rstrip('=') | |
# unicode/str python2 issues | |
return (digest, str(length)) # type: ignore | |
def open_for_csv(name, mode): | |
# type: (str, Text) -> IO[Any] | |
if sys.version_info[0] < 3: | |
nl = {} # type: Dict[str, Any] | |
bin = 'b' | |
else: | |
nl = {'newline': ''} # type: Dict[str, Any] | |
bin = '' | |
return open(name, mode + bin, **nl) | |
def fix_script(path): | |
# type: (str) -> Optional[bool] | |
"""Replace #!python with #!/path/to/python | |
Return True if file was changed. | |
""" | |
# XXX RECORD hashes will need to be updated | |
if os.path.isfile(path): | |
with open(path, 'rb') as script: | |
firstline = script.readline() | |
if not firstline.startswith(b'#!python'): | |
return False | |
exename = sys.executable.encode(sys.getfilesystemencoding()) | |
firstline = b'#!' + exename + os.linesep.encode("ascii") | |
rest = script.read() | |
with open(path, 'wb') as script: | |
script.write(firstline) | |
script.write(rest) | |
return True | |
return None | |
def wheel_root_is_purelib(metadata): | |
# type: (Message) -> bool | |
return metadata.get("Root-Is-Purelib", "").lower() == "true" | |
def get_entrypoints(filename): | |
# type: (str) -> Tuple[Dict[str, str], Dict[str, str]] | |
if not os.path.exists(filename): | |
return {}, {} | |
# This is done because you can pass a string to entry_points wrappers which | |
# means that they may or may not be valid INI files. The attempt here is to | |
# strip leading and trailing whitespace in order to make them valid INI | |
# files. | |
with open(filename) as fp: | |
data = StringIO() | |
for line in fp: | |
data.write(line.strip()) | |
data.write("\n") | |
data.seek(0) | |
# get the entry points and then the script names | |
entry_points = pkg_resources.EntryPoint.parse_map(data) | |
console = entry_points.get('console_scripts', {}) | |
gui = entry_points.get('gui_scripts', {}) | |
def _split_ep(s): | |
# type: (pkg_resources.EntryPoint) -> Tuple[str, str] | |
"""get the string representation of EntryPoint, | |
remove space and split on '=' | |
""" | |
split_parts = str(s).replace(" ", "").split("=") | |
return split_parts[0], split_parts[1] | |
# convert the EntryPoint objects into strings with module:function | |
console = dict(_split_ep(v) for v in console.values()) | |
gui = dict(_split_ep(v) for v in gui.values()) | |
return console, gui | |
def message_about_scripts_not_on_PATH(scripts): | |
# type: (Sequence[str]) -> Optional[str] | |
"""Determine if any scripts are not on PATH and format a warning. | |
Returns a warning message if one or more scripts are not on PATH, | |
otherwise None. | |
""" | |
if not scripts: | |
return None | |
# Group scripts by the path they were installed in | |
grouped_by_dir = collections.defaultdict(set) # type: Dict[str, Set[str]] | |
for destfile in scripts: | |
parent_dir = os.path.dirname(destfile) | |
script_name = os.path.basename(destfile) | |
grouped_by_dir[parent_dir].add(script_name) | |
# We don't want to warn for directories that are on PATH. | |
not_warn_dirs = [ | |
os.path.normcase(i).rstrip(os.sep) for i in | |
os.environ.get("PATH", "").split(os.pathsep) | |
] | |
# If an executable sits with sys.executable, we don't warn for it. | |
# This covers the case of venv invocations without activating the venv. | |
not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) | |
warn_for = { | |
parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() | |
if os.path.normcase(parent_dir) not in not_warn_dirs | |
} # type: Dict[str, Set[str]] | |
if not warn_for: | |
return None | |
# Format a message | |
msg_lines = [] | |
for parent_dir, dir_scripts in warn_for.items(): | |
sorted_scripts = sorted(dir_scripts) # type: List[str] | |
if len(sorted_scripts) == 1: | |
start_text = "script {} is".format(sorted_scripts[0]) | |
else: | |
start_text = "scripts {} are".format( | |
", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1] | |
) | |
msg_lines.append( | |
"The {} installed in '{}' which is not on PATH." | |
.format(start_text, parent_dir) | |
) | |
last_line_fmt = ( | |
"Consider adding {} to PATH or, if you prefer " | |
"to suppress this warning, use --no-warn-script-location." | |
) | |
if len(msg_lines) == 1: | |
msg_lines.append(last_line_fmt.format("this directory")) | |
else: | |
msg_lines.append(last_line_fmt.format("these directories")) | |
# Add a note if any directory starts with ~ | |
warn_for_tilde = any( | |
i[0] == "~" for i in os.environ.get("PATH", "").split(os.pathsep) if i | |
) | |
if warn_for_tilde: | |
tilde_warning_msg = ( | |
"NOTE: The current PATH contains path(s) starting with `~`, " | |
"which may not be expanded by all applications." | |
) | |
msg_lines.append(tilde_warning_msg) | |
# Returns the formatted multiline message | |
return "\n".join(msg_lines) | |
def sorted_outrows(outrows): | |
# type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow] | |
"""Return the given rows of a RECORD file in sorted order. | |
Each row is a 3-tuple (path, hash, size) and corresponds to a record of | |
a RECORD file (see PEP 376 and PEP 427 for details). For the rows | |
passed to this function, the size can be an integer as an int or string, | |
or the empty string. | |
""" | |
# Normally, there should only be one row per path, in which case the | |
# second and third elements don't come into play when sorting. | |
# However, in cases in the wild where a path might happen to occur twice, | |
# we don't want the sort operation to trigger an error (but still want | |
# determinism). Since the third element can be an int or string, we | |
# coerce each element to a string to avoid a TypeError in this case. | |
# For additional background, see-- | |
# https://github.com/pypa/pip/issues/5868 | |
return sorted(outrows, key=lambda row: tuple(str(x) for x in row)) | |
def get_csv_rows_for_installed( | |
old_csv_rows, # type: Iterable[List[str]] | |
installed, # type: Dict[str, str] | |
changed, # type: Set[str] | |
generated, # type: List[str] | |
lib_dir, # type: str | |
): | |
# type: (...) -> List[InstalledCSVRow] | |
""" | |
:param installed: A map from archive RECORD path to installation RECORD | |
path. | |
""" | |
installed_rows = [] # type: List[InstalledCSVRow] | |
for row in old_csv_rows: | |
if len(row) > 3: | |
logger.warning( | |
'RECORD line has more than three elements: {}'.format(row) | |
) | |
# Make a copy because we are mutating the row. | |
row = list(row) | |
old_path = row[0] | |
new_path = installed.pop(old_path, old_path) | |
row[0] = new_path | |
if new_path in changed: | |
digest, length = rehash(new_path) | |
row[1] = digest | |
row[2] = length | |
installed_rows.append(tuple(row)) | |
for f in generated: | |
digest, length = rehash(f) | |
installed_rows.append((normpath(f, lib_dir), digest, str(length))) | |
for f in installed: | |
installed_rows.append((installed[f], '', '')) | |
return installed_rows | |
class MissingCallableSuffix(Exception): | |
pass | |
def _raise_for_invalid_entrypoint(specification): | |
# type: (str) -> None | |
entry = get_export_entry(specification) | |
if entry is not None and entry.suffix is None: | |
raise MissingCallableSuffix(str(entry)) | |
class PipScriptMaker(ScriptMaker): | |
def make(self, specification, options=None): | |
# type: (str, Dict[str, Any]) -> List[str] | |
_raise_for_invalid_entrypoint(specification) | |
return super(PipScriptMaker, self).make(specification, options) | |
def install_unpacked_wheel( | |
name, # type: str | |
wheeldir, # type: str | |
wheel_zip, # type: ZipFile | |
scheme, # type: Scheme | |
req_description, # type: str | |
pycompile=True, # type: bool | |
warn_script_location=True # type: bool | |
): | |
# type: (...) -> None | |
"""Install a wheel. | |
:param name: Name of the project to install | |
:param wheeldir: Base directory of the unpacked wheel | |
:param wheel_zip: open ZipFile for wheel being installed | |
:param scheme: Distutils scheme dictating the install directories | |
:param req_description: String used in place of the requirement, for | |
logging | |
:param pycompile: Whether to byte-compile installed Python files | |
:param warn_script_location: Whether to check that scripts are installed | |
into a directory on PATH | |
:raises UnsupportedWheel: | |
* when the directory holds an unpacked wheel with incompatible | |
Wheel-Version | |
* when the .dist-info dir does not match the wheel | |
""" | |
# TODO: Investigate and break this up. | |
# TODO: Look into moving this into a dedicated class for representing an | |
# installation. | |
source = wheeldir.rstrip(os.path.sep) + os.path.sep | |
info_dir, metadata = parse_wheel(wheel_zip, name) | |
if wheel_root_is_purelib(metadata): | |
lib_dir = scheme.purelib | |
else: | |
lib_dir = scheme.platlib | |
subdirs = os.listdir(source) | |
data_dirs = [s for s in subdirs if s.endswith('.data')] | |
# Record details of the files moved | |
# installed = files copied from the wheel to the destination | |
# changed = files changed while installing (scripts #! line typically) | |
# generated = files newly generated during the install (script wrappers) | |
installed = {} # type: Dict[str, str] | |
changed = set() | |
generated = [] # type: List[str] | |
# Compile all of the pyc files that we're going to be installing | |
if pycompile: | |
with captured_stdout() as stdout: | |
with warnings.catch_warnings(): | |
warnings.filterwarnings('ignore') | |
compileall.compile_dir(source, force=True, quiet=True) | |
logger.debug(stdout.getvalue()) | |
def record_installed(srcfile, destfile, modified=False): | |
# type: (str, str, bool) -> None | |
"""Map archive RECORD paths to installation RECORD paths.""" | |
oldpath = normpath(srcfile, wheeldir) | |
newpath = normpath(destfile, lib_dir) | |
installed[oldpath] = newpath | |
if modified: | |
changed.add(destfile) | |
def clobber( | |
source, # type: str | |
dest, # type: str | |
is_base, # type: bool | |
fixer=None, # type: Optional[Callable[[str], Any]] | |
filter=None # type: Optional[Callable[[str], bool]] | |
): | |
# type: (...) -> None | |
ensure_dir(dest) # common for the 'include' path | |
for dir, subdirs, files in os.walk(source): | |
basedir = dir[len(source):].lstrip(os.path.sep) | |
destdir = os.path.join(dest, basedir) | |
if is_base and basedir == '': | |
subdirs[:] = [s for s in subdirs if not s.endswith('.data')] | |
for f in files: | |
# Skip unwanted files | |
if filter and filter(f): | |
continue | |
srcfile = os.path.join(dir, f) | |
destfile = os.path.join(dest, basedir, f) | |
# directory creation is lazy and after the file filtering above | |
# to ensure we don't install empty dirs; empty dirs can't be | |
# uninstalled. | |
ensure_dir(destdir) | |
# copyfile (called below) truncates the destination if it | |
# exists and then writes the new contents. This is fine in most | |
# cases, but can cause a segfault if pip has loaded a shared | |
# object (e.g. from pyopenssl through its vendored urllib3) | |
# Since the shared object is mmap'd an attempt to call a | |
# symbol in it will then cause a segfault. Unlinking the file | |
# allows writing of new contents while allowing the process to | |
# continue to use the old copy. | |
if os.path.exists(destfile): | |
os.unlink(destfile) | |
# We use copyfile (not move, copy, or copy2) to be extra sure | |
# that we are not moving directories over (copyfile fails for | |
# directories) as well as to ensure that we are not copying | |
# over any metadata because we want more control over what | |
# metadata we actually copy over. | |
shutil.copyfile(srcfile, destfile) | |
# Copy over the metadata for the file, currently this only | |
# includes the atime and mtime. | |
st = os.stat(srcfile) | |
if hasattr(os, "utime"): | |
os.utime(destfile, (st.st_atime, st.st_mtime)) | |
# If our file is executable, then make our destination file | |
# executable. | |
if os.access(srcfile, os.X_OK): | |
st = os.stat(srcfile) | |
permissions = ( | |
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | |
) | |
os.chmod(destfile, permissions) | |
changed = False | |
if fixer: | |
changed = fixer(destfile) | |
record_installed(srcfile, destfile, changed) | |
clobber(source, lib_dir, True) | |
dest_info_dir = os.path.join(lib_dir, info_dir) | |
# Get the defined entry points | |
ep_file = os.path.join(dest_info_dir, 'entry_points.txt') | |
console, gui = get_entrypoints(ep_file) | |
def is_entrypoint_wrapper(name): | |
# type: (str) -> bool | |
# EP, EP.exe and EP-script.py are scripts generated for | |
# entry point EP by setuptools | |
if name.lower().endswith('.exe'): | |
matchname = name[:-4] | |
elif name.lower().endswith('-script.py'): | |
matchname = name[:-10] | |
elif name.lower().endswith(".pya"): | |
matchname = name[:-4] | |
else: | |
matchname = name | |
# Ignore setuptools-generated scripts | |
return (matchname in console or matchname in gui) | |
for datadir in data_dirs: | |
fixer = None | |
filter = None | |
for subdir in os.listdir(os.path.join(wheeldir, datadir)): | |
fixer = None | |
if subdir == 'scripts': | |
fixer = fix_script | |
filter = is_entrypoint_wrapper | |
source = os.path.join(wheeldir, datadir, subdir) | |
dest = getattr(scheme, subdir) | |
clobber(source, dest, False, fixer=fixer, filter=filter) | |
maker = PipScriptMaker(None, scheme.scripts) | |
# Ensure old scripts are overwritten. | |
# See https://github.com/pypa/pip/issues/1800 | |
maker.clobber = True | |
# Ensure we don't generate any variants for scripts because this is almost | |
# never what somebody wants. | |
# See https://bitbucket.org/pypa/distlib/issue/35/ | |
maker.variants = {''} | |
# This is required because otherwise distlib creates scripts that are not | |
# executable. | |
# See https://bitbucket.org/pypa/distlib/issue/32/ | |
maker.set_mode = True | |
scripts_to_generate = [] | |
# Special case pip and setuptools to generate versioned wrappers | |
# | |
# The issue is that some projects (specifically, pip and setuptools) use | |
# code in setup.py to create "versioned" entry points - pip2.7 on Python | |
# 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into | |
# the wheel metadata at build time, and so if the wheel is installed with | |
# a *different* version of Python the entry points will be wrong. The | |
# correct fix for this is to enhance the metadata to be able to describe | |
# such versioned entry points, but that won't happen till Metadata 2.0 is | |
# available. | |
# In the meantime, projects using versioned entry points will either have | |
# incorrect versioned entry points, or they will not be able to distribute | |
# "universal" wheels (i.e., they will need a wheel per Python version). | |
# | |
# Because setuptools and pip are bundled with _ensurepip and virtualenv, | |
# we need to use universal wheels. So, as a stopgap until Metadata 2.0, we | |
# override the versioned entry points in the wheel and generate the | |
# correct ones. This code is purely a short-term measure until Metadata 2.0 | |
# is available. | |
# | |
# To add the level of hack in this section of code, in order to support | |
# ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment | |
# variable which will control which version scripts get installed. | |
# | |
# ENSUREPIP_OPTIONS=altinstall | |
# - Only pipX.Y and easy_install-X.Y will be generated and installed | |
# ENSUREPIP_OPTIONS=install | |
# - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note | |
# that this option is technically if ENSUREPIP_OPTIONS is set and is | |
# not altinstall | |
# DEFAULT | |
# - The default behavior is to install pip, pipX, pipX.Y, easy_install | |
# and easy_install-X.Y. | |
pip_script = console.pop('pip', None) | |
if pip_script: | |
if "ENSUREPIP_OPTIONS" not in os.environ: | |
scripts_to_generate.append('pip = ' + pip_script) | |
if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": | |
scripts_to_generate.append( | |
'pip%s = %s' % (sys.version_info[0], pip_script) | |
) | |
scripts_to_generate.append( | |
'pip%s = %s' % (get_major_minor_version(), pip_script) | |
) | |
# Delete any other versioned pip entry points | |
pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] | |
for k in pip_ep: | |
del console[k] | |
easy_install_script = console.pop('easy_install', None) | |
if easy_install_script: | |
if "ENSUREPIP_OPTIONS" not in os.environ: | |
scripts_to_generate.append( | |
'easy_install = ' + easy_install_script | |
) | |
scripts_to_generate.append( | |
'easy_install-%s = %s' % ( | |
get_major_minor_version(), easy_install_script | |
) | |
) | |
# Delete any other versioned easy_install entry points | |
easy_install_ep = [ | |
k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) | |
] | |
for k in easy_install_ep: | |
del console[k] | |
# Generate the console and GUI entry points specified in the wheel | |
scripts_to_generate.extend( | |
'%s = %s' % kv for kv in console.items() | |
) | |
gui_scripts_to_generate = [ | |
'%s = %s' % kv for kv in gui.items() | |
] | |
generated_console_scripts = [] # type: List[str] | |
try: | |
generated_console_scripts = maker.make_multiple(scripts_to_generate) | |
generated.extend(generated_console_scripts) | |
generated.extend( | |
maker.make_multiple(gui_scripts_to_generate, {'gui': True}) | |
) | |
except MissingCallableSuffix as e: | |
entry = e.args[0] | |
raise InstallationError( | |
"Invalid script entry point: {} for req: {} - A callable " | |
"suffix is required. Cf https://packaging.python.org/" | |
"specifications/entry-points/#use-for-scripts for more " | |
"information.".format(entry, req_description) | |
) | |
if warn_script_location: | |
msg = message_about_scripts_not_on_PATH(generated_console_scripts) | |
if msg is not None: | |
logger.warning(msg) | |
# Record pip as the installer | |
installer = os.path.join(dest_info_dir, 'INSTALLER') | |
temp_installer = os.path.join(dest_info_dir, 'INSTALLER.pip') | |
with open(temp_installer, 'wb') as installer_file: | |
installer_file.write(b'pip\n') | |
shutil.move(temp_installer, installer) | |
generated.append(installer) | |
# Record details of all files installed | |
record = os.path.join(dest_info_dir, 'RECORD') | |
temp_record = os.path.join(dest_info_dir, 'RECORD.pip') | |
with open_for_csv(record, 'r') as record_in: | |
with open_for_csv(temp_record, 'w+') as record_out: | |
reader = csv.reader(record_in) | |
outrows = get_csv_rows_for_installed( | |
reader, installed=installed, changed=changed, | |
generated=generated, lib_dir=lib_dir, | |
) | |
writer = csv.writer(record_out) | |
# Sort to simplify testing. | |
for row in sorted_outrows(outrows): | |
writer.writerow(row) | |
shutil.move(temp_record, record) | |
def install_wheel( | |
name, # type: str | |
wheel_path, # type: str | |
scheme, # type: Scheme | |
req_description, # type: str | |
pycompile=True, # type: bool | |
warn_script_location=True, # type: bool | |
_temp_dir_for_testing=None, # type: Optional[str] | |
): | |
# type: (...) -> None | |
with TempDirectory( | |
path=_temp_dir_for_testing, kind="unpacked-wheel" | |
) as unpacked_dir, ZipFile(wheel_path, allowZip64=True) as z: | |
unpack_file(wheel_path, unpacked_dir.path) | |
install_unpacked_wheel( | |
name=name, | |
wheeldir=unpacked_dir.path, | |
wheel_zip=z, | |
scheme=scheme, | |
req_description=req_description, | |
pycompile=pycompile, | |
warn_script_location=warn_script_location, | |
) |
"""Prepares a distribution for installation | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import logging | |
import mimetypes | |
import os | |
import shutil | |
import sys | |
from pip._vendor import requests | |
from pip._vendor.six import PY2 | |
from pip._internal.distributions import ( | |
make_distribution_for_install_requirement, | |
) | |
from pip._internal.distributions.installed import InstalledDistribution | |
from pip._internal.exceptions import ( | |
DirectoryUrlHashUnsupported, | |
HashMismatch, | |
HashUnpinned, | |
InstallationError, | |
PreviousBuildDirError, | |
VcsHashUnsupported, | |
) | |
from pip._internal.utils.filesystem import copy2_fixed | |
from pip._internal.utils.hashes import MissingHashes | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.marker_files import write_delete_marker_file | |
from pip._internal.utils.misc import ( | |
ask_path_exists, | |
backup_dir, | |
display_path, | |
hide_url, | |
path_to_display, | |
rmtree, | |
) | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.unpacking import unpack_file | |
from pip._internal.vcs import vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Callable, List, Optional, Tuple, | |
) | |
from mypy_extensions import TypedDict | |
from pip._internal.distributions import AbstractDistribution | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.models.link import Link | |
from pip._internal.network.download import Downloader | |
from pip._internal.req.req_install import InstallRequirement | |
from pip._internal.req.req_tracker import RequirementTracker | |
from pip._internal.utils.hashes import Hashes | |
if PY2: | |
CopytreeKwargs = TypedDict( | |
'CopytreeKwargs', | |
{ | |
'ignore': Callable[[str, List[str]], List[str]], | |
'symlinks': bool, | |
}, | |
total=False, | |
) | |
else: | |
CopytreeKwargs = TypedDict( | |
'CopytreeKwargs', | |
{ | |
'copy_function': Callable[[str, str], None], | |
'ignore': Callable[[str, List[str]], List[str]], | |
'ignore_dangling_symlinks': bool, | |
'symlinks': bool, | |
}, | |
total=False, | |
) | |
logger = logging.getLogger(__name__) | |
def _get_prepared_distribution( | |
req, # type: InstallRequirement | |
req_tracker, # type: RequirementTracker | |
finder, # type: PackageFinder | |
build_isolation # type: bool | |
): | |
# type: (...) -> AbstractDistribution | |
"""Prepare a distribution for installation. | |
""" | |
abstract_dist = make_distribution_for_install_requirement(req) | |
with req_tracker.track(req): | |
abstract_dist.prepare_distribution_metadata(finder, build_isolation) | |
return abstract_dist | |
def unpack_vcs_link(link, location): | |
# type: (Link, str) -> None | |
vcs_backend = vcs.get_backend_for_scheme(link.scheme) | |
assert vcs_backend is not None | |
vcs_backend.unpack(location, url=hide_url(link.url)) | |
def _copy_file(filename, location, link): | |
# type: (str, str, Link) -> None | |
copy = True | |
download_location = os.path.join(location, link.filename) | |
if os.path.exists(download_location): | |
response = ask_path_exists( | |
'The file {} exists. (i)gnore, (w)ipe, (b)ackup, (a)abort'.format( | |
display_path(download_location) | |
), | |
('i', 'w', 'b', 'a'), | |
) | |
if response == 'i': | |
copy = False | |
elif response == 'w': | |
logger.warning('Deleting %s', display_path(download_location)) | |
os.remove(download_location) | |
elif response == 'b': | |
dest_file = backup_dir(download_location) | |
logger.warning( | |
'Backing up %s to %s', | |
display_path(download_location), | |
display_path(dest_file), | |
) | |
shutil.move(download_location, dest_file) | |
elif response == 'a': | |
sys.exit(-1) | |
if copy: | |
shutil.copy(filename, download_location) | |
logger.info('Saved %s', display_path(download_location)) | |
def unpack_http_url( | |
link, # type: Link | |
location, # type: str | |
downloader, # type: Downloader | |
download_dir=None, # type: Optional[str] | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> str | |
temp_dir = TempDirectory(kind="unpack", globally_managed=True) | |
# If a download dir is specified, is the file already downloaded there? | |
already_downloaded_path = None | |
if download_dir: | |
already_downloaded_path = _check_download_dir( | |
link, download_dir, hashes | |
) | |
if already_downloaded_path: | |
from_path = already_downloaded_path | |
content_type = mimetypes.guess_type(from_path)[0] | |
else: | |
# let's download to a tmp dir | |
from_path, content_type = _download_http_url( | |
link, downloader, temp_dir.path, hashes | |
) | |
# unpack the archive to the build dir location. even when only | |
# downloading archives, they have to be unpacked to parse dependencies | |
unpack_file(from_path, location, content_type) | |
return from_path | |
def _copy2_ignoring_special_files(src, dest): | |
# type: (str, str) -> None | |
"""Copying special files is not supported, but as a convenience to users | |
we skip errors copying them. This supports tools that may create e.g. | |
socket files in the project source directory. | |
""" | |
try: | |
copy2_fixed(src, dest) | |
except shutil.SpecialFileError as e: | |
# SpecialFileError may be raised due to either the source or | |
# destination. If the destination was the cause then we would actually | |
# care, but since the destination directory is deleted prior to | |
# copy we ignore all of them assuming it is caused by the source. | |
logger.warning( | |
"Ignoring special file error '%s' encountered copying %s to %s.", | |
str(e), | |
path_to_display(src), | |
path_to_display(dest), | |
) | |
def _copy_source_tree(source, target): | |
# type: (str, str) -> None | |
def ignore(d, names): | |
# type: (str, List[str]) -> List[str] | |
# Pulling in those directories can potentially be very slow, | |
# exclude the following directories if they appear in the top | |
# level dir (and only it). | |
# See discussion at https://github.com/pypa/pip/pull/6770 | |
return ['.tox', '.nox'] if d == source else [] | |
kwargs = dict(ignore=ignore, symlinks=True) # type: CopytreeKwargs | |
if not PY2: | |
# Python 2 does not support copy_function, so we only ignore | |
# errors on special file copy in Python 3. | |
kwargs['copy_function'] = _copy2_ignoring_special_files | |
shutil.copytree(source, target, **kwargs) | |
def unpack_file_url( | |
link, # type: Link | |
location, # type: str | |
download_dir=None, # type: Optional[str] | |
hashes=None # type: Optional[Hashes] | |
): | |
# type: (...) -> Optional[str] | |
"""Unpack link into location. | |
""" | |
link_path = link.file_path | |
# If it's a url to a local directory | |
if link.is_existing_dir(): | |
if os.path.isdir(location): | |
rmtree(location) | |
_copy_source_tree(link_path, location) | |
return None | |
# If a download dir is specified, is the file already there and valid? | |
already_downloaded_path = None | |
if download_dir: | |
already_downloaded_path = _check_download_dir( | |
link, download_dir, hashes | |
) | |
if already_downloaded_path: | |
from_path = already_downloaded_path | |
else: | |
from_path = link_path | |
# If --require-hashes is off, `hashes` is either empty, the | |
# link's embedded hash, or MissingHashes; it is required to | |
# match. If --require-hashes is on, we are satisfied by any | |
# hash in `hashes` matching: a URL-based or an option-based | |
# one; no internet-sourced hash will be in `hashes`. | |
if hashes: | |
hashes.check_against_path(from_path) | |
content_type = mimetypes.guess_type(from_path)[0] | |
# unpack the archive to the build dir location. even when only downloading | |
# archives, they have to be unpacked to parse dependencies | |
unpack_file(from_path, location, content_type) | |
return from_path | |
def unpack_url( | |
link, # type: Link | |
location, # type: str | |
downloader, # type: Downloader | |
download_dir=None, # type: Optional[str] | |
hashes=None, # type: Optional[Hashes] | |
): | |
# type: (...) -> Optional[str] | |
"""Unpack link into location, downloading if required. | |
:param hashes: A Hashes object, one of whose embedded hashes must match, | |
or HashMismatch will be raised. If the Hashes is empty, no matches are | |
required, and unhashable types of requirements (like VCS ones, which | |
would ordinarily raise HashUnsupported) are allowed. | |
""" | |
# non-editable vcs urls | |
if link.is_vcs: | |
unpack_vcs_link(link, location) | |
return None | |
# file urls | |
elif link.is_file: | |
return unpack_file_url(link, location, download_dir, hashes=hashes) | |
# http urls | |
else: | |
return unpack_http_url( | |
link, | |
location, | |
downloader, | |
download_dir, | |
hashes=hashes, | |
) | |
def _download_http_url( | |
link, # type: Link | |
downloader, # type: Downloader | |
temp_dir, # type: str | |
hashes, # type: Optional[Hashes] | |
): | |
# type: (...) -> Tuple[str, str] | |
"""Download link url into temp_dir using provided session""" | |
download = downloader(link) | |
file_path = os.path.join(temp_dir, download.filename) | |
with open(file_path, 'wb') as content_file: | |
for chunk in download.chunks: | |
content_file.write(chunk) | |
if hashes: | |
hashes.check_against_path(file_path) | |
return file_path, download.response.headers.get('content-type', '') | |
def _check_download_dir(link, download_dir, hashes): | |
# type: (Link, str, Optional[Hashes]) -> Optional[str] | |
""" Check download_dir for previously downloaded file with correct hash | |
If a correct file is found return its path else None | |
""" | |
download_path = os.path.join(download_dir, link.filename) | |
if not os.path.exists(download_path): | |
return None | |
# If already downloaded, does its hash match? | |
logger.info('File was already downloaded %s', download_path) | |
if hashes: | |
try: | |
hashes.check_against_path(download_path) | |
except HashMismatch: | |
logger.warning( | |
'Previously-downloaded file %s has bad hash. ' | |
'Re-downloading.', | |
download_path | |
) | |
os.unlink(download_path) | |
return None | |
return download_path | |
class RequirementPreparer(object): | |
"""Prepares a Requirement | |
""" | |
def __init__( | |
self, | |
build_dir, # type: str | |
download_dir, # type: Optional[str] | |
src_dir, # type: str | |
wheel_download_dir, # type: Optional[str] | |
build_isolation, # type: bool | |
req_tracker, # type: RequirementTracker | |
downloader, # type: Downloader | |
finder, # type: PackageFinder | |
require_hashes, # type: bool | |
use_user_site, # type: bool | |
): | |
# type: (...) -> None | |
super(RequirementPreparer, self).__init__() | |
self.src_dir = src_dir | |
self.build_dir = build_dir | |
self.req_tracker = req_tracker | |
self.downloader = downloader | |
self.finder = finder | |
# Where still-packed archives should be written to. If None, they are | |
# not saved, and are deleted immediately after unpacking. | |
self.download_dir = download_dir | |
# Where still-packed .whl files should be written to. If None, they are | |
# written to the download_dir parameter. Separate to download_dir to | |
# permit only keeping wheel archives for pip wheel. | |
self.wheel_download_dir = wheel_download_dir | |
# NOTE | |
# download_dir and wheel_download_dir overlap semantically and may | |
# be combined if we're willing to have non-wheel archives present in | |
# the wheelhouse output by 'pip wheel'. | |
# Is build isolation allowed? | |
self.build_isolation = build_isolation | |
# Should hash-checking be required? | |
self.require_hashes = require_hashes | |
# Should install in user site-packages? | |
self.use_user_site = use_user_site | |
@property | |
def _download_should_save(self): | |
# type: () -> bool | |
if not self.download_dir: | |
return False | |
if os.path.exists(self.download_dir): | |
return True | |
logger.critical('Could not find download directory') | |
raise InstallationError( | |
"Could not find or access download directory '{}'" | |
.format(self.download_dir)) | |
def prepare_linked_requirement( | |
self, | |
req, # type: InstallRequirement | |
): | |
# type: (...) -> AbstractDistribution | |
"""Prepare a requirement that would be obtained from req.link | |
""" | |
assert req.link | |
link = req.link | |
# TODO: Breakup into smaller functions | |
if link.scheme == 'file': | |
path = link.file_path | |
logger.info('Processing %s', display_path(path)) | |
else: | |
logger.info('Collecting %s', req.req or req) | |
with indent_log(): | |
# @@ if filesystem packages are not marked | |
# editable in a req, a non deterministic error | |
# occurs when the script attempts to unpack the | |
# build directory | |
# Since source_dir is only set for editable requirements. | |
assert req.source_dir is None | |
req.ensure_has_source_dir(self.build_dir) | |
# If a checkout exists, it's unwise to keep going. version | |
# inconsistencies are logged later, but do not fail the | |
# installation. | |
# FIXME: this won't upgrade when there's an existing | |
# package unpacked in `req.source_dir` | |
if os.path.exists(os.path.join(req.source_dir, 'setup.py')): | |
raise PreviousBuildDirError( | |
"pip can't proceed with requirements '{}' due to a" | |
" pre-existing build directory ({}). This is " | |
"likely due to a previous installation that failed" | |
". pip is being responsible and not assuming it " | |
"can delete this. Please delete it and try again." | |
.format(req, req.source_dir) | |
) | |
# Now that we have the real link, we can tell what kind of | |
# requirements we have and raise some more informative errors | |
# than otherwise. (For example, we can raise VcsHashUnsupported | |
# for a VCS URL rather than HashMissing.) | |
if self.require_hashes: | |
# We could check these first 2 conditions inside | |
# unpack_url and save repetition of conditions, but then | |
# we would report less-useful error messages for | |
# unhashable requirements, complaining that there's no | |
# hash provided. | |
if link.is_vcs: | |
raise VcsHashUnsupported() | |
elif link.is_existing_dir(): | |
raise DirectoryUrlHashUnsupported() | |
if not req.original_link and not req.is_pinned: | |
# Unpinned packages are asking for trouble when a new | |
# version is uploaded. This isn't a security check, but | |
# it saves users a surprising hash mismatch in the | |
# future. | |
# | |
# file:/// URLs aren't pinnable, so don't complain | |
# about them not being pinned. | |
raise HashUnpinned() | |
hashes = req.hashes(trust_internet=not self.require_hashes) | |
if self.require_hashes and not hashes: | |
# Known-good hashes are missing for this requirement, so | |
# shim it with a facade object that will provoke hash | |
# computation and then raise a HashMissing exception | |
# showing the user what the hash should be. | |
hashes = MissingHashes() | |
download_dir = self.download_dir | |
if link.is_wheel and self.wheel_download_dir: | |
# when doing 'pip wheel` we download wheels to a | |
# dedicated dir. | |
download_dir = self.wheel_download_dir | |
try: | |
local_path = unpack_url( | |
link, req.source_dir, self.downloader, download_dir, | |
hashes=hashes, | |
) | |
except requests.HTTPError as exc: | |
logger.critical( | |
'Could not install requirement %s because of error %s', | |
req, | |
exc, | |
) | |
raise InstallationError( | |
'Could not install requirement {} because of HTTP ' | |
'error {} for URL {}'.format(req, exc, link) | |
) | |
# For use in later processing, preserve the file path on the | |
# requirement. | |
if local_path: | |
req.local_file_path = local_path | |
if link.is_wheel: | |
if download_dir: | |
# When downloading, we only unpack wheels to get | |
# metadata. | |
autodelete_unpacked = True | |
else: | |
# When installing a wheel, we use the unpacked | |
# wheel. | |
autodelete_unpacked = False | |
else: | |
# We always delete unpacked sdists after pip runs. | |
autodelete_unpacked = True | |
if autodelete_unpacked: | |
write_delete_marker_file(req.source_dir) | |
abstract_dist = _get_prepared_distribution( | |
req, self.req_tracker, self.finder, self.build_isolation, | |
) | |
if download_dir: | |
if link.is_existing_dir(): | |
logger.info('Link is a directory, ignoring download_dir') | |
elif local_path and not os.path.exists( | |
os.path.join(download_dir, link.filename) | |
): | |
_copy_file(local_path, download_dir, link) | |
if self._download_should_save: | |
# Make a .zip of the source_dir we already created. | |
if link.is_vcs: | |
req.archive(self.download_dir) | |
return abstract_dist | |
def prepare_editable_requirement( | |
self, | |
req, # type: InstallRequirement | |
): | |
# type: (...) -> AbstractDistribution | |
"""Prepare an editable requirement | |
""" | |
assert req.editable, "cannot prepare a non-editable req as editable" | |
logger.info('Obtaining %s', req) | |
with indent_log(): | |
if self.require_hashes: | |
raise InstallationError( | |
'The editable requirement {} cannot be installed when ' | |
'requiring hashes, because there is no single file to ' | |
'hash.'.format(req) | |
) | |
req.ensure_has_source_dir(self.src_dir) | |
req.update_editable(not self._download_should_save) | |
abstract_dist = _get_prepared_distribution( | |
req, self.req_tracker, self.finder, self.build_isolation, | |
) | |
if self._download_should_save: | |
req.archive(self.download_dir) | |
req.check_if_exists(self.use_user_site) | |
return abstract_dist | |
def prepare_installed_requirement( | |
self, | |
req, # type: InstallRequirement | |
skip_reason # type: str | |
): | |
# type: (...) -> AbstractDistribution | |
"""Prepare an already-installed requirement | |
""" | |
assert req.satisfied_by, "req should have been satisfied but isn't" | |
assert skip_reason is not None, ( | |
"did not get skip reason skipped but req.satisfied_by " | |
"is set to {}".format(req.satisfied_by) | |
) | |
logger.info( | |
'Requirement %s: %s (%s)', | |
skip_reason, req, req.satisfied_by.version | |
) | |
with indent_log(): | |
if self.require_hashes: | |
logger.debug( | |
'Since it is already installed, we are trusting this ' | |
'package without checking its hash. To ensure a ' | |
'completely repeatable environment, install into an ' | |
'empty virtualenv.' | |
) | |
abstract_dist = InstalledDistribution(req) | |
return abstract_dist |
from __future__ import absolute_import | |
import io | |
import os | |
import sys | |
from collections import namedtuple | |
from pip._vendor import pytoml, six | |
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Optional, List | |
def _is_list_of_str(obj): | |
# type: (Any) -> bool | |
return ( | |
isinstance(obj, list) and | |
all(isinstance(item, six.string_types) for item in obj) | |
) | |
def make_pyproject_path(unpacked_source_directory): | |
# type: (str) -> str | |
path = os.path.join(unpacked_source_directory, 'pyproject.toml') | |
# Python2 __file__ should not be unicode | |
if six.PY2 and isinstance(path, six.text_type): | |
path = path.encode(sys.getfilesystemencoding()) | |
return path | |
BuildSystemDetails = namedtuple('BuildSystemDetails', [ | |
'requires', 'backend', 'check', 'backend_path' | |
]) | |
def load_pyproject_toml( | |
use_pep517, # type: Optional[bool] | |
pyproject_toml, # type: str | |
setup_py, # type: str | |
req_name # type: str | |
): | |
# type: (...) -> Optional[BuildSystemDetails] | |
"""Load the pyproject.toml file. | |
Parameters: | |
use_pep517 - Has the user requested PEP 517 processing? None | |
means the user hasn't explicitly specified. | |
pyproject_toml - Location of the project's pyproject.toml file | |
setup_py - Location of the project's setup.py file | |
req_name - The name of the requirement we're processing (for | |
error reporting) | |
Returns: | |
None if we should use the legacy code path, otherwise a tuple | |
( | |
requirements from pyproject.toml, | |
name of PEP 517 backend, | |
requirements we should check are installed after setting | |
up the build environment | |
directory paths to import the backend from (backend-path), | |
relative to the project root. | |
) | |
""" | |
has_pyproject = os.path.isfile(pyproject_toml) | |
has_setup = os.path.isfile(setup_py) | |
if has_pyproject: | |
with io.open(pyproject_toml, encoding="utf-8") as f: | |
pp_toml = pytoml.load(f) | |
build_system = pp_toml.get("build-system") | |
else: | |
build_system = None | |
# The following cases must use PEP 517 | |
# We check for use_pep517 being non-None and falsey because that means | |
# the user explicitly requested --no-use-pep517. The value 0 as | |
# opposed to False can occur when the value is provided via an | |
# environment variable or config file option (due to the quirk of | |
# strtobool() returning an integer in pip's configuration code). | |
if has_pyproject and not has_setup: | |
if use_pep517 is not None and not use_pep517: | |
raise InstallationError( | |
"Disabling PEP 517 processing is invalid: " | |
"project does not have a setup.py" | |
) | |
use_pep517 = True | |
elif build_system and "build-backend" in build_system: | |
if use_pep517 is not None and not use_pep517: | |
raise InstallationError( | |
"Disabling PEP 517 processing is invalid: " | |
"project specifies a build backend of {} " | |
"in pyproject.toml".format( | |
build_system["build-backend"] | |
) | |
) | |
use_pep517 = True | |
# If we haven't worked out whether to use PEP 517 yet, | |
# and the user hasn't explicitly stated a preference, | |
# we do so if the project has a pyproject.toml file. | |
elif use_pep517 is None: | |
use_pep517 = has_pyproject | |
# At this point, we know whether we're going to use PEP 517. | |
assert use_pep517 is not None | |
# If we're using the legacy code path, there is nothing further | |
# for us to do here. | |
if not use_pep517: | |
return None | |
if build_system is None: | |
# Either the user has a pyproject.toml with no build-system | |
# section, or the user has no pyproject.toml, but has opted in | |
# explicitly via --use-pep517. | |
# In the absence of any explicit backend specification, we | |
# assume the setuptools backend that most closely emulates the | |
# traditional direct setup.py execution, and require wheel and | |
# a version of setuptools that supports that backend. | |
build_system = { | |
"requires": ["setuptools>=40.8.0", "wheel"], | |
"build-backend": "setuptools.build_meta:__legacy__", | |
} | |
# If we're using PEP 517, we have build system information (either | |
# from pyproject.toml, or defaulted by the code above). | |
# Note that at this point, we do not know if the user has actually | |
# specified a backend, though. | |
assert build_system is not None | |
# Ensure that the build-system section in pyproject.toml conforms | |
# to PEP 518. | |
error_template = ( | |
"{package} has a pyproject.toml file that does not comply " | |
"with PEP 518: {reason}" | |
) | |
# Specifying the build-system table but not the requires key is invalid | |
if "requires" not in build_system: | |
raise InstallationError( | |
error_template.format(package=req_name, reason=( | |
"it has a 'build-system' table but not " | |
"'build-system.requires' which is mandatory in the table" | |
)) | |
) | |
# Error out if requires is not a list of strings | |
requires = build_system["requires"] | |
if not _is_list_of_str(requires): | |
raise InstallationError(error_template.format( | |
package=req_name, | |
reason="'build-system.requires' is not a list of strings.", | |
)) | |
# Each requirement must be valid as per PEP 508 | |
for requirement in requires: | |
try: | |
Requirement(requirement) | |
except InvalidRequirement: | |
raise InstallationError( | |
error_template.format( | |
package=req_name, | |
reason=( | |
"'build-system.requires' contains an invalid " | |
"requirement: {!r}".format(requirement) | |
), | |
) | |
) | |
backend = build_system.get("build-backend") | |
backend_path = build_system.get("backend-path", []) | |
check = [] # type: List[str] | |
if backend is None: | |
# If the user didn't specify a backend, we assume they want to use | |
# the setuptools backend. But we can't be sure they have included | |
# a version of setuptools which supplies the backend, or wheel | |
# (which is needed by the backend) in their requirements. So we | |
# make a note to check that those requirements are present once | |
# we have set up the environment. | |
# This is quite a lot of work to check for a very specific case. But | |
# the problem is, that case is potentially quite common - projects that | |
# adopted PEP 518 early for the ability to specify requirements to | |
# execute setup.py, but never considered needing to mention the build | |
# tools themselves. The original PEP 518 code had a similar check (but | |
# implemented in a different way). | |
backend = "setuptools.build_meta:__legacy__" | |
check = ["setuptools>=40.8.0", "wheel"] | |
return BuildSystemDetails(requires, backend, check, backend_path) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from .req_file import parse_requirements | |
from .req_install import InstallRequirement | |
from .req_set import RequirementSet | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, List, Sequence | |
__all__ = [ | |
"RequirementSet", "InstallRequirement", | |
"parse_requirements", "install_given_reqs", | |
] | |
logger = logging.getLogger(__name__) | |
class InstallationResult(object): | |
def __init__(self, name): | |
# type: (str) -> None | |
self.name = name | |
def __repr__(self): | |
# type: () -> str | |
return "InstallationResult(name={!r})".format(self.name) | |
def install_given_reqs( | |
to_install, # type: List[InstallRequirement] | |
install_options, # type: List[str] | |
global_options=(), # type: Sequence[str] | |
*args, # type: Any | |
**kwargs # type: Any | |
): | |
# type: (...) -> List[InstallationResult] | |
""" | |
Install everything in the given list. | |
(to be called after having downloaded and unpacked the packages) | |
""" | |
if to_install: | |
logger.info( | |
'Installing collected packages: %s', | |
', '.join([req.name for req in to_install]), | |
) | |
installed = [] | |
with indent_log(): | |
for requirement in to_install: | |
if requirement.should_reinstall: | |
logger.info('Attempting uninstall: %s', requirement.name) | |
with indent_log(): | |
uninstalled_pathset = requirement.uninstall( | |
auto_confirm=True | |
) | |
try: | |
requirement.install( | |
install_options, | |
global_options, | |
*args, | |
**kwargs | |
) | |
except Exception: | |
should_rollback = ( | |
requirement.should_reinstall and | |
not requirement.install_succeeded | |
) | |
# if install did not succeed, rollback previous uninstall | |
if should_rollback: | |
uninstalled_pathset.rollback() | |
raise | |
else: | |
should_commit = ( | |
requirement.should_reinstall and | |
requirement.install_succeeded | |
) | |
if should_commit: | |
uninstalled_pathset.commit() | |
installed.append(InstallationResult(requirement.name)) | |
return installed |
"""Backing implementation for InstallRequirement's various constructors | |
The idea here is that these formed a major chunk of InstallRequirement's size | |
so, moving them and support code dedicated to them outside of that class | |
helps creates for better understandability for the rest of the code. | |
These are meant to be used elsewhere within pip to create instances of | |
InstallRequirement. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import logging | |
import os | |
import re | |
from pip._vendor.packaging.markers import Marker | |
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement | |
from pip._vendor.packaging.specifiers import Specifier | |
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.models.index import PyPI, TestPyPI | |
from pip._internal.models.link import Link | |
from pip._internal.models.wheel import Wheel | |
from pip._internal.pyproject import make_pyproject_path | |
from pip._internal.req.req_install import InstallRequirement | |
from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS | |
from pip._internal.utils.misc import is_installable_dir, splitext | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url | |
from pip._internal.vcs import is_url, vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Dict, Optional, Set, Tuple, Union, | |
) | |
from pip._internal.cache import WheelCache | |
__all__ = [ | |
"install_req_from_editable", "install_req_from_line", | |
"parse_editable" | |
] | |
logger = logging.getLogger(__name__) | |
operators = Specifier._operators.keys() | |
def is_archive_file(name): | |
# type: (str) -> bool | |
"""Return True if `name` is a considered as an archive file.""" | |
ext = splitext(name)[1].lower() | |
if ext in ARCHIVE_EXTENSIONS: | |
return True | |
return False | |
def _strip_extras(path): | |
# type: (str) -> Tuple[str, Optional[str]] | |
m = re.match(r'^(.+)(\[[^\]]+\])$', path) | |
extras = None | |
if m: | |
path_no_extras = m.group(1) | |
extras = m.group(2) | |
else: | |
path_no_extras = path | |
return path_no_extras, extras | |
def convert_extras(extras): | |
# type: (Optional[str]) -> Set[str] | |
if not extras: | |
return set() | |
return Requirement("placeholder" + extras.lower()).extras | |
def parse_editable(editable_req): | |
# type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] | |
"""Parses an editable requirement into: | |
- a requirement name | |
- an URL | |
- extras | |
- editable options | |
Accepted requirements: | |
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir | |
.[some_extra] | |
""" | |
url = editable_req | |
# If a file path is specified with extras, strip off the extras. | |
url_no_extras, extras = _strip_extras(url) | |
if os.path.isdir(url_no_extras): | |
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): | |
msg = ( | |
'File "setup.py" not found. Directory cannot be installed ' | |
'in editable mode: {}'.format(os.path.abspath(url_no_extras)) | |
) | |
pyproject_path = make_pyproject_path(url_no_extras) | |
if os.path.isfile(pyproject_path): | |
msg += ( | |
'\n(A "pyproject.toml" file was found, but editable ' | |
'mode currently requires a setup.py based build.)' | |
) | |
raise InstallationError(msg) | |
# Treating it as code that has already been checked out | |
url_no_extras = path_to_url(url_no_extras) | |
if url_no_extras.lower().startswith('file:'): | |
package_name = Link(url_no_extras).egg_fragment | |
if extras: | |
return ( | |
package_name, | |
url_no_extras, | |
Requirement("placeholder" + extras.lower()).extras, | |
) | |
else: | |
return package_name, url_no_extras, None | |
for version_control in vcs: | |
if url.lower().startswith('%s:' % version_control): | |
url = '%s+%s' % (version_control, url) | |
break | |
if '+' not in url: | |
raise InstallationError( | |
'{} is not a valid editable requirement. ' | |
'It should either be a path to a local project or a VCS URL ' | |
'(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req) | |
) | |
vc_type = url.split('+', 1)[0].lower() | |
if not vcs.get_backend(vc_type): | |
error_message = 'For --editable=%s only ' % editable_req + \ | |
', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ | |
' is currently supported' | |
raise InstallationError(error_message) | |
package_name = Link(url).egg_fragment | |
if not package_name: | |
raise InstallationError( | |
"Could not detect requirement name for '%s', please specify one " | |
"with #egg=your_package_name" % editable_req | |
) | |
return package_name, url, None | |
def deduce_helpful_msg(req): | |
# type: (str) -> str | |
"""Returns helpful msg in case requirements file does not exist, | |
or cannot be parsed. | |
:params req: Requirements file path | |
""" | |
msg = "" | |
if os.path.exists(req): | |
msg = " It does exist." | |
# Try to parse and check if it is a requirements file. | |
try: | |
with open(req, 'r') as fp: | |
# parse first line only | |
next(parse_requirements(fp.read())) | |
msg += " The argument you provided " + \ | |
"(%s) appears to be a" % (req) + \ | |
" requirements file. If that is the" + \ | |
" case, use the '-r' flag to install" + \ | |
" the packages specified within it." | |
except RequirementParseError: | |
logger.debug("Cannot parse '%s' as requirements \ | |
file" % (req), exc_info=True) | |
else: | |
msg += " File '%s' does not exist." % (req) | |
return msg | |
class RequirementParts(object): | |
def __init__( | |
self, | |
requirement, # type: Optional[Requirement] | |
link, # type: Optional[Link] | |
markers, # type: Optional[Marker] | |
extras, # type: Set[str] | |
): | |
self.requirement = requirement | |
self.link = link | |
self.markers = markers | |
self.extras = extras | |
def parse_req_from_editable(editable_req): | |
# type: (str) -> RequirementParts | |
name, url, extras_override = parse_editable(editable_req) | |
if name is not None: | |
try: | |
req = Requirement(name) | |
except InvalidRequirement: | |
raise InstallationError("Invalid requirement: '%s'" % name) | |
else: | |
req = None | |
link = Link(url) | |
return RequirementParts(req, link, None, extras_override) | |
# ---- The actual constructors follow ---- | |
def install_req_from_editable( | |
editable_req, # type: str | |
comes_from=None, # type: Optional[str] | |
use_pep517=None, # type: Optional[bool] | |
isolated=False, # type: bool | |
options=None, # type: Optional[Dict[str, Any]] | |
wheel_cache=None, # type: Optional[WheelCache] | |
constraint=False # type: bool | |
): | |
# type: (...) -> InstallRequirement | |
parts = parse_req_from_editable(editable_req) | |
source_dir = parts.link.file_path if parts.link.scheme == 'file' else None | |
return InstallRequirement( | |
parts.requirement, comes_from, source_dir=source_dir, | |
editable=True, | |
link=parts.link, | |
constraint=constraint, | |
use_pep517=use_pep517, | |
isolated=isolated, | |
options=options if options else {}, | |
wheel_cache=wheel_cache, | |
extras=parts.extras, | |
) | |
def _looks_like_path(name): | |
# type: (str) -> bool | |
"""Checks whether the string "looks like" a path on the filesystem. | |
This does not check whether the target actually exists, only judge from the | |
appearance. | |
Returns true if any of the following conditions is true: | |
* a path separator is found (either os.path.sep or os.path.altsep); | |
* a dot is found (which represents the current directory). | |
""" | |
if os.path.sep in name: | |
return True | |
if os.path.altsep is not None and os.path.altsep in name: | |
return True | |
if name.startswith("."): | |
return True | |
return False | |
def _get_url_from_path(path, name): | |
# type: (str, str) -> str | |
""" | |
First, it checks whether a provided path is an installable directory | |
(e.g. it has a setup.py). If it is, returns the path. | |
If false, check if the path is an archive file (such as a .whl). | |
The function checks if the path is a file. If false, if the path has | |
an @, it will treat it as a PEP 440 URL requirement and return the path. | |
""" | |
if _looks_like_path(name) and os.path.isdir(path): | |
if is_installable_dir(path): | |
return path_to_url(path) | |
raise InstallationError( | |
"Directory %r is not installable. Neither 'setup.py' " | |
"nor 'pyproject.toml' found." % name | |
) | |
if not is_archive_file(path): | |
return None | |
if os.path.isfile(path): | |
return path_to_url(path) | |
urlreq_parts = name.split('@', 1) | |
if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): | |
# If the path contains '@' and the part before it does not look | |
# like a path, try to treat it as a PEP 440 URL req instead. | |
return None | |
logger.warning( | |
'Requirement %r looks like a filename, but the ' | |
'file does not exist', | |
name | |
) | |
return path_to_url(path) | |
def parse_req_from_line(name, line_source): | |
# type: (str, Optional[str]) -> RequirementParts | |
if is_url(name): | |
marker_sep = '; ' | |
else: | |
marker_sep = ';' | |
if marker_sep in name: | |
name, markers_as_string = name.split(marker_sep, 1) | |
markers_as_string = markers_as_string.strip() | |
if not markers_as_string: | |
markers = None | |
else: | |
markers = Marker(markers_as_string) | |
else: | |
markers = None | |
name = name.strip() | |
req_as_string = None | |
path = os.path.normpath(os.path.abspath(name)) | |
link = None | |
extras_as_string = None | |
if is_url(name): | |
link = Link(name) | |
else: | |
p, extras_as_string = _strip_extras(path) | |
url = _get_url_from_path(p, name) | |
if url is not None: | |
link = Link(url) | |
# it's a local file, dir, or url | |
if link: | |
# Handle relative file URLs | |
if link.scheme == 'file' and re.search(r'\.\./', link.url): | |
link = Link( | |
path_to_url(os.path.normpath(os.path.abspath(link.path)))) | |
# wheel file | |
if link.is_wheel: | |
wheel = Wheel(link.filename) # can raise InvalidWheelFilename | |
req_as_string = "%s==%s" % (wheel.name, wheel.version) | |
else: | |
# set the req to the egg fragment. when it's not there, this | |
# will become an 'unnamed' requirement | |
req_as_string = link.egg_fragment | |
# a requirement specifier | |
else: | |
req_as_string = name | |
extras = convert_extras(extras_as_string) | |
def with_source(text): | |
# type: (str) -> str | |
if not line_source: | |
return text | |
return '{} (from {})'.format(text, line_source) | |
if req_as_string is not None: | |
try: | |
req = Requirement(req_as_string) | |
except InvalidRequirement: | |
if os.path.sep in req_as_string: | |
add_msg = "It looks like a path." | |
add_msg += deduce_helpful_msg(req_as_string) | |
elif ('=' in req_as_string and | |
not any(op in req_as_string for op in operators)): | |
add_msg = "= is not a valid operator. Did you mean == ?" | |
else: | |
add_msg = '' | |
msg = with_source( | |
'Invalid requirement: {!r}'.format(req_as_string) | |
) | |
if add_msg: | |
msg += '\nHint: {}'.format(add_msg) | |
raise InstallationError(msg) | |
else: | |
req = None | |
return RequirementParts(req, link, markers, extras) | |
def install_req_from_line( | |
name, # type: str | |
comes_from=None, # type: Optional[Union[str, InstallRequirement]] | |
use_pep517=None, # type: Optional[bool] | |
isolated=False, # type: bool | |
options=None, # type: Optional[Dict[str, Any]] | |
wheel_cache=None, # type: Optional[WheelCache] | |
constraint=False, # type: bool | |
line_source=None, # type: Optional[str] | |
): | |
# type: (...) -> InstallRequirement | |
"""Creates an InstallRequirement from a name, which might be a | |
requirement, directory containing 'setup.py', filename, or URL. | |
:param line_source: An optional string describing where the line is from, | |
for logging purposes in case of an error. | |
""" | |
parts = parse_req_from_line(name, line_source) | |
return InstallRequirement( | |
parts.requirement, comes_from, link=parts.link, markers=parts.markers, | |
use_pep517=use_pep517, isolated=isolated, | |
options=options if options else {}, | |
wheel_cache=wheel_cache, | |
constraint=constraint, | |
extras=parts.extras, | |
) | |
def install_req_from_req_string( | |
req_string, # type: str | |
comes_from=None, # type: Optional[InstallRequirement] | |
isolated=False, # type: bool | |
wheel_cache=None, # type: Optional[WheelCache] | |
use_pep517=None # type: Optional[bool] | |
): | |
# type: (...) -> InstallRequirement | |
try: | |
req = Requirement(req_string) | |
except InvalidRequirement: | |
raise InstallationError("Invalid requirement: '%s'" % req_string) | |
domains_not_allowed = [ | |
PyPI.file_storage_domain, | |
TestPyPI.file_storage_domain, | |
] | |
if (req.url and comes_from and comes_from.link and | |
comes_from.link.netloc in domains_not_allowed): | |
# Explicitly disallow pypi packages that depend on external urls | |
raise InstallationError( | |
"Packages installed from PyPI cannot depend on packages " | |
"which are not also hosted on PyPI.\n" | |
"%s depends on %s " % (comes_from.name, req) | |
) | |
return InstallRequirement( | |
req, comes_from, isolated=isolated, wheel_cache=wheel_cache, | |
use_pep517=use_pep517 | |
) |
""" | |
Requirements file parsing | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import optparse | |
import os | |
import re | |
import shlex | |
import sys | |
from pip._vendor.six.moves import filterfalse | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.cli import cmdoptions | |
from pip._internal.exceptions import ( | |
InstallationError, | |
RequirementsFileParseError, | |
) | |
from pip._internal.models.search_scope import SearchScope | |
from pip._internal.req.constructors import ( | |
install_req_from_editable, | |
install_req_from_line, | |
) | |
from pip._internal.utils.encoding import auto_decode | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import get_url_scheme | |
if MYPY_CHECK_RUNNING: | |
from optparse import Values | |
from typing import ( | |
Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple, | |
) | |
from pip._internal.req import InstallRequirement | |
from pip._internal.cache import WheelCache | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.network.session import PipSession | |
ReqFileLines = Iterator[Tuple[int, Text]] | |
LineParser = Callable[[Text], Tuple[str, Values]] | |
__all__ = ['parse_requirements'] | |
SCHEME_RE = re.compile(r'^(http|https|file):', re.I) | |
COMMENT_RE = re.compile(r'(^|\s+)#.*$') | |
# Matches environment variable-style values in '${MY_VARIABLE_1}' with the | |
# variable name consisting of only uppercase letters, digits or the '_' | |
# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, | |
# 2013 Edition. | |
ENV_VAR_RE = re.compile(r'(?P<var>\$\{(?P<name>[A-Z0-9_]+)\})') | |
SUPPORTED_OPTIONS = [ | |
cmdoptions.index_url, | |
cmdoptions.extra_index_url, | |
cmdoptions.no_index, | |
cmdoptions.constraints, | |
cmdoptions.requirements, | |
cmdoptions.editable, | |
cmdoptions.find_links, | |
cmdoptions.no_binary, | |
cmdoptions.only_binary, | |
cmdoptions.require_hashes, | |
cmdoptions.pre, | |
cmdoptions.trusted_host, | |
cmdoptions.always_unzip, # Deprecated | |
] # type: List[Callable[..., optparse.Option]] | |
# options to be passed to requirements | |
SUPPORTED_OPTIONS_REQ = [ | |
cmdoptions.install_options, | |
cmdoptions.global_options, | |
cmdoptions.hash, | |
] # type: List[Callable[..., optparse.Option]] | |
# the 'dest' string values | |
SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] | |
class ParsedLine(object): | |
def __init__( | |
self, | |
filename, # type: str | |
lineno, # type: int | |
comes_from, # type: str | |
args, # type: str | |
opts, # type: Values | |
constraint, # type: bool | |
): | |
# type: (...) -> None | |
self.filename = filename | |
self.lineno = lineno | |
self.comes_from = comes_from | |
self.args = args | |
self.opts = opts | |
self.constraint = constraint | |
def parse_requirements( | |
filename, # type: str | |
session, # type: PipSession | |
finder=None, # type: Optional[PackageFinder] | |
comes_from=None, # type: Optional[str] | |
options=None, # type: Optional[optparse.Values] | |
constraint=False, # type: bool | |
wheel_cache=None, # type: Optional[WheelCache] | |
use_pep517=None # type: Optional[bool] | |
): | |
# type: (...) -> Iterator[InstallRequirement] | |
"""Parse a requirements file and yield InstallRequirement instances. | |
:param filename: Path or url of requirements file. | |
:param session: PipSession instance. | |
:param finder: Instance of pip.index.PackageFinder. | |
:param comes_from: Origin description of requirements. | |
:param options: cli options. | |
:param constraint: If true, parsing a constraint file rather than | |
requirements file. | |
:param wheel_cache: Instance of pip.wheel.WheelCache | |
:param use_pep517: Value of the --use-pep517 option. | |
""" | |
skip_requirements_regex = ( | |
options.skip_requirements_regex if options else None | |
) | |
line_parser = get_line_parser(finder) | |
parser = RequirementsFileParser( | |
session, line_parser, comes_from, skip_requirements_regex | |
) | |
for parsed_line in parser.parse(filename, constraint): | |
req = handle_line( | |
parsed_line, finder, options, session, wheel_cache, use_pep517 | |
) | |
if req is not None: | |
yield req | |
def preprocess(content, skip_requirements_regex): | |
# type: (Text, Optional[str]) -> ReqFileLines | |
"""Split, filter, and join lines, and return a line iterator | |
:param content: the content of the requirements file | |
:param options: cli options | |
""" | |
lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines | |
lines_enum = join_lines(lines_enum) | |
lines_enum = ignore_comments(lines_enum) | |
if skip_requirements_regex: | |
lines_enum = skip_regex(lines_enum, skip_requirements_regex) | |
lines_enum = expand_env_variables(lines_enum) | |
return lines_enum | |
def handle_line( | |
line, # type: ParsedLine | |
finder=None, # type: Optional[PackageFinder] | |
options=None, # type: Optional[optparse.Values] | |
session=None, # type: Optional[PipSession] | |
wheel_cache=None, # type: Optional[WheelCache] | |
use_pep517=None, # type: Optional[bool] | |
): | |
# type: (...) -> Optional[InstallRequirement] | |
"""Handle a single parsed requirements line; This can result in | |
creating/yielding requirements, or updating the finder. | |
For lines that contain requirements, the only options that have an effect | |
are from SUPPORTED_OPTIONS_REQ, and they are scoped to the | |
requirement. Other options from SUPPORTED_OPTIONS may be present, but are | |
ignored. | |
For lines that do not contain requirements, the only options that have an | |
effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may | |
be present, but are ignored. These lines may contain multiple options | |
(although our docs imply only one is supported), and all our parsed and | |
affect the finder. | |
""" | |
# preserve for the nested code path | |
line_comes_from = '%s %s (line %s)' % ( | |
'-c' if line.constraint else '-r', line.filename, line.lineno, | |
) | |
# return a line requirement | |
if line.args: | |
isolated = options.isolated_mode if options else False | |
if options: | |
cmdoptions.check_install_build_global(options, line.opts) | |
# get the options that apply to requirements | |
req_options = {} | |
for dest in SUPPORTED_OPTIONS_REQ_DEST: | |
if dest in line.opts.__dict__ and line.opts.__dict__[dest]: | |
req_options[dest] = line.opts.__dict__[dest] | |
line_source = 'line {} of {}'.format(line.lineno, line.filename) | |
return install_req_from_line( | |
line.args, | |
comes_from=line_comes_from, | |
use_pep517=use_pep517, | |
isolated=isolated, | |
options=req_options, | |
wheel_cache=wheel_cache, | |
constraint=line.constraint, | |
line_source=line_source, | |
) | |
# return an editable requirement | |
elif line.opts.editables: | |
isolated = options.isolated_mode if options else False | |
return install_req_from_editable( | |
line.opts.editables[0], comes_from=line_comes_from, | |
use_pep517=use_pep517, | |
constraint=line.constraint, isolated=isolated, | |
wheel_cache=wheel_cache | |
) | |
# percolate hash-checking option upward | |
elif line.opts.require_hashes: | |
options.require_hashes = line.opts.require_hashes | |
# set finder options | |
elif finder: | |
find_links = finder.find_links | |
index_urls = finder.index_urls | |
if line.opts.index_url: | |
index_urls = [line.opts.index_url] | |
if line.opts.no_index is True: | |
index_urls = [] | |
if line.opts.extra_index_urls: | |
index_urls.extend(line.opts.extra_index_urls) | |
if line.opts.find_links: | |
# FIXME: it would be nice to keep track of the source | |
# of the find_links: support a find-links local path | |
# relative to a requirements file. | |
value = line.opts.find_links[0] | |
req_dir = os.path.dirname(os.path.abspath(line.filename)) | |
relative_to_reqs_file = os.path.join(req_dir, value) | |
if os.path.exists(relative_to_reqs_file): | |
value = relative_to_reqs_file | |
find_links.append(value) | |
search_scope = SearchScope( | |
find_links=find_links, | |
index_urls=index_urls, | |
) | |
finder.search_scope = search_scope | |
if line.opts.pre: | |
finder.set_allow_all_prereleases() | |
if session: | |
for host in line.opts.trusted_hosts or []: | |
source = 'line {} of {}'.format(line.lineno, line.filename) | |
session.add_trusted_host(host, source=source) | |
return None | |
class RequirementsFileParser(object): | |
def __init__( | |
self, | |
session, # type: PipSession | |
line_parser, # type: LineParser | |
comes_from, # type: str | |
skip_requirements_regex, # type: Optional[str] | |
): | |
# type: (...) -> None | |
self._session = session | |
self._line_parser = line_parser | |
self._comes_from = comes_from | |
self._skip_requirements_regex = skip_requirements_regex | |
def parse(self, filename, constraint): | |
# type: (str, bool) -> Iterator[ParsedLine] | |
"""Parse a given file, yielding parsed lines. | |
""" | |
for line in self._parse_and_recurse(filename, constraint): | |
yield line | |
def _parse_and_recurse(self, filename, constraint): | |
# type: (str, bool) -> Iterator[ParsedLine] | |
for line in self._parse_file(filename, constraint): | |
if ( | |
not line.args and | |
not line.opts.editables and | |
(line.opts.requirements or line.opts.constraints) | |
): | |
# parse a nested requirements file | |
if line.opts.requirements: | |
req_path = line.opts.requirements[0] | |
nested_constraint = False | |
else: | |
req_path = line.opts.constraints[0] | |
nested_constraint = True | |
# original file is over http | |
if SCHEME_RE.search(filename): | |
# do a url join so relative paths work | |
req_path = urllib_parse.urljoin(filename, req_path) | |
# original file and nested file are paths | |
elif not SCHEME_RE.search(req_path): | |
# do a join so relative paths work | |
req_path = os.path.join( | |
os.path.dirname(filename), req_path, | |
) | |
for inner_line in self._parse_and_recurse( | |
req_path, nested_constraint, | |
): | |
yield inner_line | |
else: | |
yield line | |
def _parse_file(self, filename, constraint): | |
# type: (str, bool) -> Iterator[ParsedLine] | |
_, content = get_file_content( | |
filename, self._session, comes_from=self._comes_from | |
) | |
lines_enum = preprocess(content, self._skip_requirements_regex) | |
for line_number, line in lines_enum: | |
try: | |
args_str, opts = self._line_parser(line) | |
except OptionParsingError as e: | |
# add offending line | |
msg = 'Invalid requirement: %s\n%s' % (line, e.msg) | |
raise RequirementsFileParseError(msg) | |
yield ParsedLine( | |
filename, | |
line_number, | |
self._comes_from, | |
args_str, | |
opts, | |
constraint, | |
) | |
def get_line_parser(finder): | |
# type: (Optional[PackageFinder]) -> LineParser | |
def parse_line(line): | |
# type: (Text) -> Tuple[str, Values] | |
# Build new parser for each line since it accumulates appendable | |
# options. | |
parser = build_parser() | |
defaults = parser.get_default_values() | |
defaults.index_url = None | |
if finder: | |
defaults.format_control = finder.format_control | |
args_str, options_str = break_args_options(line) | |
# Prior to 2.7.3, shlex cannot deal with unicode entries | |
if sys.version_info < (2, 7, 3): | |
# https://github.com/python/mypy/issues/1174 | |
options_str = options_str.encode('utf8') # type: ignore | |
# https://github.com/python/mypy/issues/1174 | |
opts, _ = parser.parse_args( | |
shlex.split(options_str), defaults) # type: ignore | |
return args_str, opts | |
return parse_line | |
def break_args_options(line): | |
# type: (Text) -> Tuple[str, Text] | |
"""Break up the line into an args and options string. We only want to shlex | |
(and then optparse) the options, not the args. args can contain markers | |
which are corrupted by shlex. | |
""" | |
tokens = line.split(' ') | |
args = [] | |
options = tokens[:] | |
for token in tokens: | |
if token.startswith('-') or token.startswith('--'): | |
break | |
else: | |
args.append(token) | |
options.pop(0) | |
return ' '.join(args), ' '.join(options) # type: ignore | |
class OptionParsingError(Exception): | |
def __init__(self, msg): | |
# type: (str) -> None | |
self.msg = msg | |
def build_parser(): | |
# type: () -> optparse.OptionParser | |
""" | |
Return a parser for parsing requirement lines | |
""" | |
parser = optparse.OptionParser(add_help_option=False) | |
option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ | |
for option_factory in option_factories: | |
option = option_factory() | |
parser.add_option(option) | |
# By default optparse sys.exits on parsing errors. We want to wrap | |
# that in our own exception. | |
def parser_exit(self, msg): | |
# type: (Any, str) -> NoReturn | |
raise OptionParsingError(msg) | |
# NOTE: mypy disallows assigning to a method | |
# https://github.com/python/mypy/issues/2427 | |
parser.exit = parser_exit # type: ignore | |
return parser | |
def join_lines(lines_enum): | |
# type: (ReqFileLines) -> ReqFileLines | |
"""Joins a line ending in '\' with the previous line (except when following | |
comments). The joined line takes on the index of the first line. | |
""" | |
primary_line_number = None | |
new_line = [] # type: List[Text] | |
for line_number, line in lines_enum: | |
if not line.endswith('\\') or COMMENT_RE.match(line): | |
if COMMENT_RE.match(line): | |
# this ensures comments are always matched later | |
line = ' ' + line | |
if new_line: | |
new_line.append(line) | |
yield primary_line_number, ''.join(new_line) | |
new_line = [] | |
else: | |
yield line_number, line | |
else: | |
if not new_line: | |
primary_line_number = line_number | |
new_line.append(line.strip('\\')) | |
# last line contains \ | |
if new_line: | |
yield primary_line_number, ''.join(new_line) | |
# TODO: handle space after '\'. | |
def ignore_comments(lines_enum): | |
# type: (ReqFileLines) -> ReqFileLines | |
""" | |
Strips comments and filter empty lines. | |
""" | |
for line_number, line in lines_enum: | |
line = COMMENT_RE.sub('', line) | |
line = line.strip() | |
if line: | |
yield line_number, line | |
def skip_regex(lines_enum, pattern): | |
# type: (ReqFileLines, str) -> ReqFileLines | |
""" | |
Skip lines that match the provided pattern | |
Note: the regex pattern is only built once | |
""" | |
matcher = re.compile(pattern) | |
lines_enum = filterfalse(lambda e: matcher.search(e[1]), lines_enum) | |
return lines_enum | |
def expand_env_variables(lines_enum): | |
# type: (ReqFileLines) -> ReqFileLines | |
"""Replace all environment variables that can be retrieved via `os.getenv`. | |
The only allowed format for environment variables defined in the | |
requirement file is `${MY_VARIABLE_1}` to ensure two things: | |
1. Strings that contain a `$` aren't accidentally (partially) expanded. | |
2. Ensure consistency across platforms for requirement files. | |
These points are the result of a discussion on the `github pull | |
request #3514 <https://github.com/pypa/pip/pull/3514>`_. | |
Valid characters in variable names follow the `POSIX standard | |
<http://pubs.opengroup.org/onlinepubs/9699919799/>`_ and are limited | |
to uppercase letter, digits and the `_` (underscore). | |
""" | |
for line_number, line in lines_enum: | |
for env_var, var_name in ENV_VAR_RE.findall(line): | |
value = os.getenv(var_name) | |
if not value: | |
continue | |
line = line.replace(env_var, value) | |
yield line_number, line | |
def get_file_content(url, session, comes_from=None): | |
# type: (str, PipSession, Optional[str]) -> Tuple[str, Text] | |
"""Gets the content of a file; it may be a filename, file: URL, or | |
http: URL. Returns (location, content). Content is unicode. | |
Respects # -*- coding: declarations on the retrieved files. | |
:param url: File path or url. | |
:param session: PipSession instance. | |
:param comes_from: Origin description of requirements. | |
""" | |
scheme = get_url_scheme(url) | |
if scheme in ['http', 'https']: | |
# FIXME: catch some errors | |
resp = session.get(url) | |
resp.raise_for_status() | |
return resp.url, resp.text | |
elif scheme == 'file': | |
if comes_from and comes_from.startswith('http'): | |
raise InstallationError( | |
'Requirements file %s references URL %s, which is local' | |
% (comes_from, url)) | |
path = url.split(':', 1)[1] | |
path = path.replace('\\', '/') | |
match = _url_slash_drive_re.match(path) | |
if match: | |
path = match.group(1) + ':' + path.split('|', 1)[1] | |
path = urllib_parse.unquote(path) | |
if path.startswith('/'): | |
path = '/' + path.lstrip('/') | |
url = path | |
try: | |
with open(url, 'rb') as f: | |
content = auto_decode(f.read()) | |
except IOError as exc: | |
raise InstallationError( | |
'Could not open requirements file: %s' % str(exc) | |
) | |
return url, content | |
_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import shutil | |
import sys | |
import zipfile | |
from pip._vendor import pkg_resources, six | |
from pip._vendor.packaging.requirements import Requirement | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.packaging.version import Version | |
from pip._vendor.packaging.version import parse as parse_version | |
from pip._vendor.pep517.wrappers import Pep517HookCaller | |
from pip._internal import pep425tags | |
from pip._internal.build_env import NoOpBuildEnvironment | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.locations import get_scheme | |
from pip._internal.models.link import Link | |
from pip._internal.operations.build.metadata import generate_metadata | |
from pip._internal.operations.build.metadata_legacy import \ | |
generate_metadata as generate_metadata_legacy | |
from pip._internal.operations.install.editable_legacy import \ | |
install_editable as install_editable_legacy | |
from pip._internal.operations.install.legacy import install as install_legacy | |
from pip._internal.operations.install.wheel import install_wheel | |
from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path | |
from pip._internal.req.req_uninstall import UninstallPathSet | |
from pip._internal.utils.deprecation import deprecated | |
from pip._internal.utils.hashes import Hashes | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.marker_files import ( | |
PIP_DELETE_MARKER_FILENAME, | |
has_delete_marker_file, | |
write_delete_marker_file, | |
) | |
from pip._internal.utils.misc import ( | |
ask_path_exists, | |
backup_dir, | |
display_path, | |
dist_in_site_packages, | |
dist_in_usersite, | |
get_installed_version, | |
hide_url, | |
redact_auth_from_url, | |
rmtree, | |
) | |
from pip._internal.utils.packaging import get_metadata | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.virtualenv import running_under_virtualenv | |
from pip._internal.vcs import vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Dict, Iterable, List, Optional, Sequence, Union, | |
) | |
from pip._internal.build_env import BuildEnvironment | |
from pip._internal.cache import WheelCache | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._vendor.pkg_resources import Distribution | |
from pip._vendor.packaging.specifiers import SpecifierSet | |
from pip._vendor.packaging.markers import Marker | |
logger = logging.getLogger(__name__) | |
def _get_dist(metadata_directory): | |
# type: (str) -> Distribution | |
"""Return a pkg_resources.Distribution for the provided | |
metadata directory. | |
""" | |
dist_dir = metadata_directory.rstrip(os.sep) | |
# Determine the correct Distribution object type. | |
if dist_dir.endswith(".egg-info"): | |
dist_cls = pkg_resources.Distribution | |
else: | |
assert dist_dir.endswith(".dist-info") | |
dist_cls = pkg_resources.DistInfoDistribution | |
# Build a PathMetadata object, from path to metadata. :wink: | |
base_dir, dist_dir_name = os.path.split(dist_dir) | |
dist_name = os.path.splitext(dist_dir_name)[0] | |
metadata = pkg_resources.PathMetadata(base_dir, dist_dir) | |
return dist_cls( | |
base_dir, | |
project_name=dist_name, | |
metadata=metadata, | |
) | |
class InstallRequirement(object): | |
""" | |
Represents something that may be installed later on, may have information | |
about where to fetch the relevant requirement and also contains logic for | |
installing the said requirement. | |
""" | |
def __init__( | |
self, | |
req, # type: Optional[Requirement] | |
comes_from, # type: Optional[Union[str, InstallRequirement]] | |
source_dir=None, # type: Optional[str] | |
editable=False, # type: bool | |
link=None, # type: Optional[Link] | |
markers=None, # type: Optional[Marker] | |
use_pep517=None, # type: Optional[bool] | |
isolated=False, # type: bool | |
options=None, # type: Optional[Dict[str, Any]] | |
wheel_cache=None, # type: Optional[WheelCache] | |
constraint=False, # type: bool | |
extras=() # type: Iterable[str] | |
): | |
# type: (...) -> None | |
assert req is None or isinstance(req, Requirement), req | |
self.req = req | |
self.comes_from = comes_from | |
self.constraint = constraint | |
if source_dir is None: | |
self.source_dir = None # type: Optional[str] | |
else: | |
self.source_dir = os.path.normpath(os.path.abspath(source_dir)) | |
self.editable = editable | |
self._wheel_cache = wheel_cache | |
if link is None and req and req.url: | |
# PEP 508 URL requirement | |
link = Link(req.url) | |
self.link = self.original_link = link | |
# Path to any downloaded or already-existing package. | |
self.local_file_path = None # type: Optional[str] | |
if self.link and self.link.is_file: | |
self.local_file_path = self.link.file_path | |
if extras: | |
self.extras = extras | |
elif req: | |
self.extras = { | |
pkg_resources.safe_extra(extra) for extra in req.extras | |
} | |
else: | |
self.extras = set() | |
if markers is None and req: | |
markers = req.marker | |
self.markers = markers | |
# This holds the pkg_resources.Distribution object if this requirement | |
# is already available: | |
self.satisfied_by = None # type: Optional[Distribution] | |
# Whether the installation process should try to uninstall an existing | |
# distribution before installing this requirement. | |
self.should_reinstall = False | |
# Temporary build location | |
self._temp_build_dir = None # type: Optional[TempDirectory] | |
# Set to True after successful installation | |
self.install_succeeded = None # type: Optional[bool] | |
self.options = options if options else {} | |
# Set to True after successful preparation of this requirement | |
self.prepared = False | |
self.is_direct = False | |
self.isolated = isolated | |
self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment | |
# For PEP 517, the directory where we request the project metadata | |
# gets stored. We need this to pass to build_wheel, so the backend | |
# can ensure that the wheel matches the metadata (see the PEP for | |
# details). | |
self.metadata_directory = None # type: Optional[str] | |
# The static build requirements (from pyproject.toml) | |
self.pyproject_requires = None # type: Optional[List[str]] | |
# Build requirements that we will check are available | |
self.requirements_to_check = [] # type: List[str] | |
# The PEP 517 backend we should use to build the project | |
self.pep517_backend = None # type: Optional[Pep517HookCaller] | |
# Are we using PEP 517 for this requirement? | |
# After pyproject.toml has been loaded, the only valid values are True | |
# and False. Before loading, None is valid (meaning "use the default"). | |
# Setting an explicit value before loading pyproject.toml is supported, | |
# but after loading this flag should be treated as read only. | |
self.use_pep517 = use_pep517 | |
def __str__(self): | |
# type: () -> str | |
if self.req: | |
s = str(self.req) | |
if self.link: | |
s += ' from %s' % redact_auth_from_url(self.link.url) | |
elif self.link: | |
s = redact_auth_from_url(self.link.url) | |
else: | |
s = '<InstallRequirement>' | |
if self.satisfied_by is not None: | |
s += ' in %s' % display_path(self.satisfied_by.location) | |
if self.comes_from: | |
if isinstance(self.comes_from, six.string_types): | |
comes_from = self.comes_from # type: Optional[str] | |
else: | |
comes_from = self.comes_from.from_path() | |
if comes_from: | |
s += ' (from %s)' % comes_from | |
return s | |
def __repr__(self): | |
# type: () -> str | |
return '<%s object: %s editable=%r>' % ( | |
self.__class__.__name__, str(self), self.editable) | |
def format_debug(self): | |
# type: () -> str | |
"""An un-tested helper for getting state, for debugging. | |
""" | |
attributes = vars(self) | |
names = sorted(attributes) | |
state = ( | |
"{}={!r}".format(attr, attributes[attr]) for attr in sorted(names) | |
) | |
return '<{name} object: {{{state}}}>'.format( | |
name=self.__class__.__name__, | |
state=", ".join(state), | |
) | |
def populate_link(self, finder, upgrade, require_hashes): | |
# type: (PackageFinder, bool, bool) -> None | |
"""Ensure that if a link can be found for this, that it is found. | |
Note that self.link may still be None - if Upgrade is False and the | |
requirement is already installed. | |
If require_hashes is True, don't use the wheel cache, because cached | |
wheels, always built locally, have different hashes than the files | |
downloaded from the index server and thus throw false hash mismatches. | |
Furthermore, cached wheels at present have undeterministic contents due | |
to file modification times. | |
""" | |
if self.link is None: | |
self.link = finder.find_requirement(self, upgrade) | |
if self._wheel_cache is not None and not require_hashes: | |
old_link = self.link | |
supported_tags = pep425tags.get_supported() | |
self.link = self._wheel_cache.get( | |
link=self.link, | |
package_name=self.name, | |
supported_tags=supported_tags, | |
) | |
if old_link != self.link: | |
logger.debug('Using cached wheel link: %s', self.link) | |
# Things that are valid for all kinds of requirements? | |
@property | |
def name(self): | |
# type: () -> Optional[str] | |
if self.req is None: | |
return None | |
return six.ensure_str(pkg_resources.safe_name(self.req.name)) | |
@property | |
def specifier(self): | |
# type: () -> SpecifierSet | |
return self.req.specifier | |
@property | |
def is_pinned(self): | |
# type: () -> bool | |
"""Return whether I am pinned to an exact version. | |
For example, some-package==1.2 is pinned; some-package>1.2 is not. | |
""" | |
specifiers = self.specifier | |
return (len(specifiers) == 1 and | |
next(iter(specifiers)).operator in {'==', '==='}) | |
@property | |
def installed_version(self): | |
# type: () -> Optional[str] | |
return get_installed_version(self.name) | |
def match_markers(self, extras_requested=None): | |
# type: (Optional[Iterable[str]]) -> bool | |
if not extras_requested: | |
# Provide an extra to safely evaluate the markers | |
# without matching any extra | |
extras_requested = ('',) | |
if self.markers is not None: | |
return any( | |
self.markers.evaluate({'extra': extra}) | |
for extra in extras_requested) | |
else: | |
return True | |
@property | |
def has_hash_options(self): | |
# type: () -> bool | |
"""Return whether any known-good hashes are specified as options. | |
These activate --require-hashes mode; hashes specified as part of a | |
URL do not. | |
""" | |
return bool(self.options.get('hashes', {})) | |
def hashes(self, trust_internet=True): | |
# type: (bool) -> Hashes | |
"""Return a hash-comparer that considers my option- and URL-based | |
hashes to be known-good. | |
Hashes in URLs--ones embedded in the requirements file, not ones | |
downloaded from an index server--are almost peers with ones from | |
flags. They satisfy --require-hashes (whether it was implicitly or | |
explicitly activated) but do not activate it. md5 and sha224 are not | |
allowed in flags, which should nudge people toward good algos. We | |
always OR all hashes together, even ones from URLs. | |
:param trust_internet: Whether to trust URL-based (#md5=...) hashes | |
downloaded from the internet, as by populate_link() | |
""" | |
good_hashes = self.options.get('hashes', {}).copy() | |
link = self.link if trust_internet else self.original_link | |
if link and link.hash: | |
good_hashes.setdefault(link.hash_name, []).append(link.hash) | |
return Hashes(good_hashes) | |
def from_path(self): | |
# type: () -> Optional[str] | |
"""Format a nice indicator to show where this "comes from" | |
""" | |
if self.req is None: | |
return None | |
s = str(self.req) | |
if self.comes_from: | |
if isinstance(self.comes_from, six.string_types): | |
comes_from = self.comes_from | |
else: | |
comes_from = self.comes_from.from_path() | |
if comes_from: | |
s += '->' + comes_from | |
return s | |
def ensure_build_location(self, build_dir): | |
# type: (str) -> str | |
assert build_dir is not None | |
if self._temp_build_dir is not None: | |
assert self._temp_build_dir.path | |
return self._temp_build_dir.path | |
if self.req is None: | |
# Some systems have /tmp as a symlink which confuses custom | |
# builds (such as numpy). Thus, we ensure that the real path | |
# is returned. | |
self._temp_build_dir = TempDirectory(kind="req-build") | |
return self._temp_build_dir.path | |
if self.editable: | |
name = self.name.lower() | |
else: | |
name = self.name | |
# FIXME: Is there a better place to create the build_dir? (hg and bzr | |
# need this) | |
if not os.path.exists(build_dir): | |
logger.debug('Creating directory %s', build_dir) | |
os.makedirs(build_dir) | |
write_delete_marker_file(build_dir) | |
return os.path.join(build_dir, name) | |
def _set_requirement(self): | |
# type: () -> None | |
"""Set requirement after generating metadata. | |
""" | |
assert self.req is None | |
assert self.metadata is not None | |
assert self.source_dir is not None | |
# Construct a Requirement object from the generated metadata | |
if isinstance(parse_version(self.metadata["Version"]), Version): | |
op = "==" | |
else: | |
op = "===" | |
self.req = Requirement( | |
"".join([ | |
self.metadata["Name"], | |
op, | |
self.metadata["Version"], | |
]) | |
) | |
def warn_on_mismatching_name(self): | |
# type: () -> None | |
metadata_name = canonicalize_name(self.metadata["Name"]) | |
if canonicalize_name(self.req.name) == metadata_name: | |
# Everything is fine. | |
return | |
# If we're here, there's a mismatch. Log a warning about it. | |
logger.warning( | |
'Generating metadata for package %s ' | |
'produced metadata for project name %s. Fix your ' | |
'#egg=%s fragments.', | |
self.name, metadata_name, self.name | |
) | |
self.req = Requirement(metadata_name) | |
def remove_temporary_source(self): | |
# type: () -> None | |
"""Remove the source files from this requirement, if they are marked | |
for deletion""" | |
if self.source_dir and has_delete_marker_file(self.source_dir): | |
logger.debug('Removing source in %s', self.source_dir) | |
rmtree(self.source_dir) | |
self.source_dir = None | |
if self._temp_build_dir: | |
self._temp_build_dir.cleanup() | |
self._temp_build_dir = None | |
self.build_env.cleanup() | |
def check_if_exists(self, use_user_site): | |
# type: (bool) -> None | |
"""Find an installed distribution that satisfies or conflicts | |
with this requirement, and set self.satisfied_by or | |
self.should_reinstall appropriately. | |
""" | |
if self.req is None: | |
return | |
# get_distribution() will resolve the entire list of requirements | |
# anyway, and we've already determined that we need the requirement | |
# in question, so strip the marker so that we don't try to | |
# evaluate it. | |
no_marker = Requirement(str(self.req)) | |
no_marker.marker = None | |
try: | |
self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) | |
except pkg_resources.DistributionNotFound: | |
return | |
except pkg_resources.VersionConflict: | |
existing_dist = pkg_resources.get_distribution( | |
self.req.name | |
) | |
if use_user_site: | |
if dist_in_usersite(existing_dist): | |
self.should_reinstall = True | |
elif (running_under_virtualenv() and | |
dist_in_site_packages(existing_dist)): | |
raise InstallationError( | |
"Will not install to the user site because it will " | |
"lack sys.path precedence to %s in %s" % | |
(existing_dist.project_name, existing_dist.location) | |
) | |
else: | |
self.should_reinstall = True | |
else: | |
if self.editable and self.satisfied_by: | |
self.should_reinstall = True | |
# when installing editables, nothing pre-existing should ever | |
# satisfy | |
self.satisfied_by = None | |
# Things valid for wheels | |
@property | |
def is_wheel(self): | |
# type: () -> bool | |
if not self.link: | |
return False | |
return self.link.is_wheel | |
# Things valid for sdists | |
@property | |
def unpacked_source_directory(self): | |
# type: () -> str | |
return os.path.join( | |
self.source_dir, | |
self.link and self.link.subdirectory_fragment or '') | |
@property | |
def setup_py_path(self): | |
# type: () -> str | |
assert self.source_dir, "No source dir for %s" % self | |
setup_py = os.path.join(self.unpacked_source_directory, 'setup.py') | |
# Python2 __file__ should not be unicode | |
if six.PY2 and isinstance(setup_py, six.text_type): | |
setup_py = setup_py.encode(sys.getfilesystemencoding()) | |
return setup_py | |
@property | |
def pyproject_toml_path(self): | |
# type: () -> str | |
assert self.source_dir, "No source dir for %s" % self | |
return make_pyproject_path(self.unpacked_source_directory) | |
def load_pyproject_toml(self): | |
# type: () -> None | |
"""Load the pyproject.toml file. | |
After calling this routine, all of the attributes related to PEP 517 | |
processing for this requirement have been set. In particular, the | |
use_pep517 attribute can be used to determine whether we should | |
follow the PEP 517 or legacy (setup.py) code path. | |
""" | |
pyproject_toml_data = load_pyproject_toml( | |
self.use_pep517, | |
self.pyproject_toml_path, | |
self.setup_py_path, | |
str(self) | |
) | |
if pyproject_toml_data is None: | |
self.use_pep517 = False | |
return | |
self.use_pep517 = True | |
requires, backend, check, backend_path = pyproject_toml_data | |
self.requirements_to_check = check | |
self.pyproject_requires = requires | |
self.pep517_backend = Pep517HookCaller( | |
self.unpacked_source_directory, backend, backend_path=backend_path, | |
) | |
def _generate_metadata(self): | |
# type: () -> str | |
"""Invokes metadata generator functions, with the required arguments. | |
""" | |
if not self.use_pep517: | |
assert self.unpacked_source_directory | |
return generate_metadata_legacy( | |
build_env=self.build_env, | |
setup_py_path=self.setup_py_path, | |
source_dir=self.unpacked_source_directory, | |
editable=self.editable, | |
isolated=self.isolated, | |
details=self.name or "from {}".format(self.link) | |
) | |
assert self.pep517_backend is not None | |
return generate_metadata( | |
build_env=self.build_env, | |
backend=self.pep517_backend, | |
) | |
def prepare_metadata(self): | |
# type: () -> None | |
"""Ensure that project metadata is available. | |
Under PEP 517, call the backend hook to prepare the metadata. | |
Under legacy processing, call setup.py egg-info. | |
""" | |
assert self.source_dir | |
with indent_log(): | |
self.metadata_directory = self._generate_metadata() | |
# Act on the newly generated metadata, based on the name and version. | |
if not self.name: | |
self._set_requirement() | |
else: | |
self.warn_on_mismatching_name() | |
self.assert_source_matches_version() | |
@property | |
def metadata(self): | |
# type: () -> Any | |
if not hasattr(self, '_metadata'): | |
self._metadata = get_metadata(self.get_dist()) | |
return self._metadata | |
def get_dist(self): | |
# type: () -> Distribution | |
return _get_dist(self.metadata_directory) | |
def assert_source_matches_version(self): | |
# type: () -> None | |
assert self.source_dir | |
version = self.metadata['version'] | |
if self.req.specifier and version not in self.req.specifier: | |
logger.warning( | |
'Requested %s, but installing version %s', | |
self, | |
version, | |
) | |
else: | |
logger.debug( | |
'Source in %s has version %s, which satisfies requirement %s', | |
display_path(self.source_dir), | |
version, | |
self, | |
) | |
# For both source distributions and editables | |
def ensure_has_source_dir(self, parent_dir): | |
# type: (str) -> None | |
"""Ensure that a source_dir is set. | |
This will create a temporary build dir if the name of the requirement | |
isn't known yet. | |
:param parent_dir: The ideal pip parent_dir for the source_dir. | |
Generally src_dir for editables and build_dir for sdists. | |
:return: self.source_dir | |
""" | |
if self.source_dir is None: | |
self.source_dir = self.ensure_build_location(parent_dir) | |
# For editable installations | |
def update_editable(self, obtain=True): | |
# type: (bool) -> None | |
if not self.link: | |
logger.debug( | |
"Cannot update repository at %s; repository location is " | |
"unknown", | |
self.source_dir, | |
) | |
return | |
assert self.editable | |
assert self.source_dir | |
if self.link.scheme == 'file': | |
# Static paths don't get updated | |
return | |
assert '+' in self.link.url, "bad url: %r" % self.link.url | |
vc_type, url = self.link.url.split('+', 1) | |
vcs_backend = vcs.get_backend(vc_type) | |
if vcs_backend: | |
if not self.link.is_vcs: | |
reason = ( | |
"This form of VCS requirement is being deprecated: {}." | |
).format( | |
self.link.url | |
) | |
replacement = None | |
if self.link.url.startswith("git+git@"): | |
replacement = ( | |
"git+https://[email protected]/..., " | |
"git+ssh://[email protected]/..., " | |
"or the insecure git+git://[email protected]/..." | |
) | |
deprecated(reason, replacement, gone_in="21.0", issue=7554) | |
hidden_url = hide_url(self.link.url) | |
if obtain: | |
vcs_backend.obtain(self.source_dir, url=hidden_url) | |
else: | |
vcs_backend.export(self.source_dir, url=hidden_url) | |
else: | |
assert 0, ( | |
'Unexpected version control type (in %s): %s' | |
% (self.link, vc_type)) | |
# Top-level Actions | |
def uninstall(self, auto_confirm=False, verbose=False): | |
# type: (bool, bool) -> Optional[UninstallPathSet] | |
""" | |
Uninstall the distribution currently satisfying this requirement. | |
Prompts before removing or modifying files unless | |
``auto_confirm`` is True. | |
Refuses to delete or modify files outside of ``sys.prefix`` - | |
thus uninstallation within a virtual environment can only | |
modify that virtual environment, even if the virtualenv is | |
linked to global site-packages. | |
""" | |
assert self.req | |
try: | |
dist = pkg_resources.get_distribution(self.req.name) | |
except pkg_resources.DistributionNotFound: | |
logger.warning("Skipping %s as it is not installed.", self.name) | |
return None | |
else: | |
logger.info('Found existing installation: %s', dist) | |
uninstalled_pathset = UninstallPathSet.from_dist(dist) | |
uninstalled_pathset.remove(auto_confirm, verbose) | |
return uninstalled_pathset | |
def _get_archive_name(self, path, parentdir, rootdir): | |
# type: (str, str, str) -> str | |
def _clean_zip_name(name, prefix): | |
# type: (str, str) -> str | |
assert name.startswith(prefix + os.path.sep), ( | |
"name %r doesn't start with prefix %r" % (name, prefix) | |
) | |
name = name[len(prefix) + 1:] | |
name = name.replace(os.path.sep, '/') | |
return name | |
path = os.path.join(parentdir, path) | |
name = _clean_zip_name(path, rootdir) | |
return self.name + '/' + name | |
def archive(self, build_dir): | |
# type: (str) -> None | |
"""Saves archive to provided build_dir. | |
Used for saving downloaded VCS requirements as part of `pip download`. | |
""" | |
assert self.source_dir | |
create_archive = True | |
archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) | |
archive_path = os.path.join(build_dir, archive_name) | |
if os.path.exists(archive_path): | |
response = ask_path_exists( | |
'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' % | |
display_path(archive_path), ('i', 'w', 'b', 'a')) | |
if response == 'i': | |
create_archive = False | |
elif response == 'w': | |
logger.warning('Deleting %s', display_path(archive_path)) | |
os.remove(archive_path) | |
elif response == 'b': | |
dest_file = backup_dir(archive_path) | |
logger.warning( | |
'Backing up %s to %s', | |
display_path(archive_path), | |
display_path(dest_file), | |
) | |
shutil.move(archive_path, dest_file) | |
elif response == 'a': | |
sys.exit(-1) | |
if not create_archive: | |
return | |
zip_output = zipfile.ZipFile( | |
archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True, | |
) | |
with zip_output: | |
dir = os.path.normcase( | |
os.path.abspath(self.unpacked_source_directory) | |
) | |
for dirpath, dirnames, filenames in os.walk(dir): | |
if 'pip-egg-info' in dirnames: | |
dirnames.remove('pip-egg-info') | |
for dirname in dirnames: | |
dir_arcname = self._get_archive_name( | |
dirname, parentdir=dirpath, rootdir=dir, | |
) | |
zipdir = zipfile.ZipInfo(dir_arcname + '/') | |
zipdir.external_attr = 0x1ED << 16 # 0o755 | |
zip_output.writestr(zipdir, '') | |
for filename in filenames: | |
if filename == PIP_DELETE_MARKER_FILENAME: | |
continue | |
file_arcname = self._get_archive_name( | |
filename, parentdir=dirpath, rootdir=dir, | |
) | |
filename = os.path.join(dirpath, filename) | |
zip_output.write(filename, file_arcname) | |
logger.info('Saved %s', display_path(archive_path)) | |
def install( | |
self, | |
install_options, # type: List[str] | |
global_options=None, # type: Optional[Sequence[str]] | |
root=None, # type: Optional[str] | |
home=None, # type: Optional[str] | |
prefix=None, # type: Optional[str] | |
warn_script_location=True, # type: bool | |
use_user_site=False, # type: bool | |
pycompile=True # type: bool | |
): | |
# type: (...) -> None | |
scheme = get_scheme( | |
self.name, | |
user=use_user_site, | |
home=home, | |
root=root, | |
isolated=self.isolated, | |
prefix=prefix, | |
) | |
global_options = global_options if global_options is not None else [] | |
if self.editable: | |
install_editable_legacy( | |
install_options, | |
global_options, | |
prefix=prefix, | |
home=home, | |
use_user_site=use_user_site, | |
name=self.name, | |
setup_py_path=self.setup_py_path, | |
isolated=self.isolated, | |
build_env=self.build_env, | |
unpacked_source_directory=self.unpacked_source_directory, | |
) | |
self.install_succeeded = True | |
return | |
if self.is_wheel: | |
assert self.local_file_path | |
install_wheel( | |
self.name, | |
self.local_file_path, | |
scheme=scheme, | |
req_description=str(self.req), | |
pycompile=pycompile, | |
warn_script_location=warn_script_location, | |
) | |
self.install_succeeded = True | |
return | |
install_legacy( | |
self, | |
install_options=install_options, | |
global_options=global_options, | |
root=root, | |
home=home, | |
prefix=prefix, | |
use_user_site=use_user_site, | |
pycompile=pycompile, | |
scheme=scheme, | |
) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
from collections import OrderedDict | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._internal import pep425tags | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.models.wheel import Wheel | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Dict, Iterable, List, Optional, Tuple | |
from pip._internal.req.req_install import InstallRequirement | |
logger = logging.getLogger(__name__) | |
class RequirementSet(object): | |
def __init__(self, check_supported_wheels=True): | |
# type: (bool) -> None | |
"""Create a RequirementSet. | |
""" | |
self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 | |
self.check_supported_wheels = check_supported_wheels | |
self.unnamed_requirements = [] # type: List[InstallRequirement] | |
self.successfully_downloaded = [] # type: List[InstallRequirement] | |
self.reqs_to_cleanup = [] # type: List[InstallRequirement] | |
def __str__(self): | |
# type: () -> str | |
requirements = sorted( | |
(req for req in self.requirements.values() if not req.comes_from), | |
key=lambda req: canonicalize_name(req.name), | |
) | |
return ' '.join(str(req.req) for req in requirements) | |
def __repr__(self): | |
# type: () -> str | |
requirements = sorted( | |
self.requirements.values(), | |
key=lambda req: canonicalize_name(req.name), | |
) | |
format_string = '<{classname} object; {count} requirement(s): {reqs}>' | |
return format_string.format( | |
classname=self.__class__.__name__, | |
count=len(requirements), | |
reqs=', '.join(str(req.req) for req in requirements), | |
) | |
def add_unnamed_requirement(self, install_req): | |
# type: (InstallRequirement) -> None | |
assert not install_req.name | |
self.unnamed_requirements.append(install_req) | |
def add_named_requirement(self, install_req): | |
# type: (InstallRequirement) -> None | |
assert install_req.name | |
project_name = canonicalize_name(install_req.name) | |
self.requirements[project_name] = install_req | |
def add_requirement( | |
self, | |
install_req, # type: InstallRequirement | |
parent_req_name=None, # type: Optional[str] | |
extras_requested=None # type: Optional[Iterable[str]] | |
): | |
# type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 | |
"""Add install_req as a requirement to install. | |
:param parent_req_name: The name of the requirement that needed this | |
added. The name is used because when multiple unnamed requirements | |
resolve to the same name, we could otherwise end up with dependency | |
links that point outside the Requirements set. parent_req must | |
already be added. Note that None implies that this is a user | |
supplied requirement, vs an inferred one. | |
:param extras_requested: an iterable of extras used to evaluate the | |
environment markers. | |
:return: Additional requirements to scan. That is either [] if | |
the requirement is not applicable, or [install_req] if the | |
requirement is applicable and has just been added. | |
""" | |
# If the markers do not match, ignore this requirement. | |
if not install_req.match_markers(extras_requested): | |
logger.info( | |
"Ignoring %s: markers '%s' don't match your environment", | |
install_req.name, install_req.markers, | |
) | |
return [], None | |
# If the wheel is not supported, raise an error. | |
# Should check this after filtering out based on environment markers to | |
# allow specifying different wheels based on the environment/OS, in a | |
# single requirements file. | |
if install_req.link and install_req.link.is_wheel: | |
wheel = Wheel(install_req.link.filename) | |
tags = pep425tags.get_supported() | |
if (self.check_supported_wheels and not wheel.supported(tags)): | |
raise InstallationError( | |
"%s is not a supported wheel on this platform." % | |
wheel.filename | |
) | |
# This next bit is really a sanity check. | |
assert install_req.is_direct == (parent_req_name is None), ( | |
"a direct req shouldn't have a parent and also, " | |
"a non direct req should have a parent" | |
) | |
# Unnamed requirements are scanned again and the requirement won't be | |
# added as a dependency until after scanning. | |
if not install_req.name: | |
self.add_unnamed_requirement(install_req) | |
return [install_req], None | |
try: | |
existing_req = self.get_requirement(install_req.name) | |
except KeyError: | |
existing_req = None | |
has_conflicting_requirement = ( | |
parent_req_name is None and | |
existing_req and | |
not existing_req.constraint and | |
existing_req.extras == install_req.extras and | |
existing_req.req.specifier != install_req.req.specifier | |
) | |
if has_conflicting_requirement: | |
raise InstallationError( | |
"Double requirement given: %s (already in %s, name=%r)" | |
% (install_req, existing_req, install_req.name) | |
) | |
# When no existing requirement exists, add the requirement as a | |
# dependency and it will be scanned again after. | |
if not existing_req: | |
self.add_named_requirement(install_req) | |
# We'd want to rescan this requirement later | |
return [install_req], install_req | |
# Assume there's no need to scan, and that we've already | |
# encountered this for scanning. | |
if install_req.constraint or not existing_req.constraint: | |
return [], existing_req | |
does_not_satisfy_constraint = ( | |
install_req.link and | |
not ( | |
existing_req.link and | |
install_req.link.path == existing_req.link.path | |
) | |
) | |
if does_not_satisfy_constraint: | |
self.reqs_to_cleanup.append(install_req) | |
raise InstallationError( | |
"Could not satisfy constraints for '%s': " | |
"installation from path or url cannot be " | |
"constrained to a version" % install_req.name, | |
) | |
# If we're now installing a constraint, mark the existing | |
# object for real installation. | |
existing_req.constraint = False | |
existing_req.extras = tuple(sorted( | |
set(existing_req.extras) | set(install_req.extras) | |
)) | |
logger.debug( | |
"Setting %s extras to: %s", | |
existing_req, existing_req.extras, | |
) | |
# Return the existing requirement for addition to the parent and | |
# scanning again. | |
return [existing_req], existing_req | |
def has_requirement(self, name): | |
# type: (str) -> bool | |
project_name = canonicalize_name(name) | |
return ( | |
project_name in self.requirements and | |
not self.requirements[project_name].constraint | |
) | |
def get_requirement(self, name): | |
# type: (str) -> InstallRequirement | |
project_name = canonicalize_name(name) | |
if project_name in self.requirements: | |
return self.requirements[project_name] | |
raise KeyError("No project with the name %r" % name) | |
def cleanup_files(self): | |
# type: () -> None | |
"""Clean up files, remove builds.""" | |
logger.debug('Cleaning up...') | |
with indent_log(): | |
for req in self.reqs_to_cleanup: | |
req.remove_temporary_source() |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import contextlib | |
import errno | |
import hashlib | |
import logging | |
import os | |
from pip._vendor import contextlib2 | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from types import TracebackType | |
from typing import Dict, Iterator, Optional, Set, Type, Union | |
from pip._internal.req.req_install import InstallRequirement | |
from pip._internal.models.link import Link | |
logger = logging.getLogger(__name__) | |
@contextlib.contextmanager | |
def update_env_context_manager(**changes): | |
# type: (str) -> Iterator[None] | |
target = os.environ | |
# Save values from the target and change them. | |
non_existent_marker = object() | |
saved_values = {} # type: Dict[str, Union[object, str]] | |
for name, new_value in changes.items(): | |
try: | |
saved_values[name] = target[name] | |
except KeyError: | |
saved_values[name] = non_existent_marker | |
target[name] = new_value | |
try: | |
yield | |
finally: | |
# Restore original values in the target. | |
for name, original_value in saved_values.items(): | |
if original_value is non_existent_marker: | |
del target[name] | |
else: | |
assert isinstance(original_value, str) # for mypy | |
target[name] = original_value | |
@contextlib.contextmanager | |
def get_requirement_tracker(): | |
# type: () -> Iterator[RequirementTracker] | |
root = os.environ.get('PIP_REQ_TRACKER') | |
with contextlib2.ExitStack() as ctx: | |
if root is None: | |
root = ctx.enter_context( | |
TempDirectory(kind='req-tracker') | |
).path | |
ctx.enter_context(update_env_context_manager(PIP_REQ_TRACKER=root)) | |
logger.debug("Initialized build tracking at %s", root) | |
with RequirementTracker(root) as tracker: | |
yield tracker | |
class RequirementTracker(object): | |
def __init__(self, root): | |
# type: (str) -> None | |
self._root = root | |
self._entries = set() # type: Set[InstallRequirement] | |
logger.debug("Created build tracker: %s", self._root) | |
def __enter__(self): | |
# type: () -> RequirementTracker | |
logger.debug("Entered build tracker: %s", self._root) | |
return self | |
def __exit__( | |
self, | |
exc_type, # type: Optional[Type[BaseException]] | |
exc_val, # type: Optional[BaseException] | |
exc_tb # type: Optional[TracebackType] | |
): | |
# type: (...) -> None | |
self.cleanup() | |
def _entry_path(self, link): | |
# type: (Link) -> str | |
hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() | |
return os.path.join(self._root, hashed) | |
def add(self, req): | |
# type: (InstallRequirement) -> None | |
"""Add an InstallRequirement to build tracking. | |
""" | |
# Get the file to write information about this requirement. | |
entry_path = self._entry_path(req.link) | |
# Try reading from the file. If it exists and can be read from, a build | |
# is already in progress, so a LookupError is raised. | |
try: | |
with open(entry_path) as fp: | |
contents = fp.read() | |
except IOError as e: | |
# if the error is anything other than "file does not exist", raise. | |
if e.errno != errno.ENOENT: | |
raise | |
else: | |
message = '%s is already being built: %s' % (req.link, contents) | |
raise LookupError(message) | |
# If we're here, req should really not be building already. | |
assert req not in self._entries | |
# Start tracking this requirement. | |
with open(entry_path, 'w') as fp: | |
fp.write(str(req)) | |
self._entries.add(req) | |
logger.debug('Added %s to build tracker %r', req, self._root) | |
def remove(self, req): | |
# type: (InstallRequirement) -> None | |
"""Remove an InstallRequirement from build tracking. | |
""" | |
# Delete the created file and the corresponding entries. | |
os.unlink(self._entry_path(req.link)) | |
self._entries.remove(req) | |
logger.debug('Removed %s from build tracker %r', req, self._root) | |
def cleanup(self): | |
# type: () -> None | |
for req in set(self._entries): | |
self.remove(req) | |
logger.debug("Removed build tracker: %r", self._root) | |
@contextlib.contextmanager | |
def track(self, req): | |
# type: (InstallRequirement) -> Iterator[None] | |
self.add(req) | |
yield | |
self.remove(req) |
from __future__ import absolute_import | |
import csv | |
import functools | |
import logging | |
import os | |
import sys | |
import sysconfig | |
from pip._vendor import pkg_resources | |
from pip._internal.exceptions import UninstallationError | |
from pip._internal.locations import bin_py, bin_user | |
from pip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import ( | |
FakeFile, | |
ask, | |
dist_in_usersite, | |
dist_is_local, | |
egg_link_path, | |
is_local, | |
normalize_path, | |
renames, | |
rmtree, | |
) | |
from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, | |
) | |
from pip._vendor.pkg_resources import Distribution | |
logger = logging.getLogger(__name__) | |
def _script_names(dist, script_name, is_gui): | |
# type: (Distribution, str, bool) -> List[str] | |
"""Create the fully qualified name of the files created by | |
{console,gui}_scripts for the given ``dist``. | |
Returns the list of file names | |
""" | |
if dist_in_usersite(dist): | |
bin_dir = bin_user | |
else: | |
bin_dir = bin_py | |
exe_name = os.path.join(bin_dir, script_name) | |
paths_to_remove = [exe_name] | |
if WINDOWS: | |
paths_to_remove.append(exe_name + '.exe') | |
paths_to_remove.append(exe_name + '.exe.manifest') | |
if is_gui: | |
paths_to_remove.append(exe_name + '-script.pyw') | |
else: | |
paths_to_remove.append(exe_name + '-script.py') | |
return paths_to_remove | |
def _unique(fn): | |
# type: (Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]] | |
@functools.wraps(fn) | |
def unique(*args, **kw): | |
# type: (Any, Any) -> Iterator[Any] | |
seen = set() # type: Set[Any] | |
for item in fn(*args, **kw): | |
if item not in seen: | |
seen.add(item) | |
yield item | |
return unique | |
@_unique | |
def uninstallation_paths(dist): | |
# type: (Distribution) -> Iterator[str] | |
""" | |
Yield all the uninstallation paths for dist based on RECORD-without-.py[co] | |
Yield paths to all the files in RECORD. For each .py file in RECORD, add | |
the .pyc and .pyo in the same directory. | |
UninstallPathSet.add() takes care of the __pycache__ .py[co]. | |
""" | |
r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD'))) | |
for row in r: | |
path = os.path.join(dist.location, row[0]) | |
yield path | |
if path.endswith('.py'): | |
dn, fn = os.path.split(path) | |
base = fn[:-3] | |
path = os.path.join(dn, base + '.pyc') | |
yield path | |
path = os.path.join(dn, base + '.pyo') | |
yield path | |
def compact(paths): | |
# type: (Iterable[str]) -> Set[str] | |
"""Compact a path set to contain the minimal number of paths | |
necessary to contain all paths in the set. If /a/path/ and | |
/a/path/to/a/file.txt are both in the set, leave only the | |
shorter path.""" | |
sep = os.path.sep | |
short_paths = set() # type: Set[str] | |
for path in sorted(paths, key=len): | |
should_skip = any( | |
path.startswith(shortpath.rstrip("*")) and | |
path[len(shortpath.rstrip("*").rstrip(sep))] == sep | |
for shortpath in short_paths | |
) | |
if not should_skip: | |
short_paths.add(path) | |
return short_paths | |
def compress_for_rename(paths): | |
# type: (Iterable[str]) -> Set[str] | |
"""Returns a set containing the paths that need to be renamed. | |
This set may include directories when the original sequence of paths | |
included every file on disk. | |
""" | |
case_map = dict((os.path.normcase(p), p) for p in paths) | |
remaining = set(case_map) | |
unchecked = sorted(set(os.path.split(p)[0] | |
for p in case_map.values()), key=len) | |
wildcards = set() # type: Set[str] | |
def norm_join(*a): | |
# type: (str) -> str | |
return os.path.normcase(os.path.join(*a)) | |
for root in unchecked: | |
if any(os.path.normcase(root).startswith(w) | |
for w in wildcards): | |
# This directory has already been handled. | |
continue | |
all_files = set() # type: Set[str] | |
all_subdirs = set() # type: Set[str] | |
for dirname, subdirs, files in os.walk(root): | |
all_subdirs.update(norm_join(root, dirname, d) | |
for d in subdirs) | |
all_files.update(norm_join(root, dirname, f) | |
for f in files) | |
# If all the files we found are in our remaining set of files to | |
# remove, then remove them from the latter set and add a wildcard | |
# for the directory. | |
if not (all_files - remaining): | |
remaining.difference_update(all_files) | |
wildcards.add(root + os.sep) | |
return set(map(case_map.__getitem__, remaining)) | wildcards | |
def compress_for_output_listing(paths): | |
# type: (Iterable[str]) -> Tuple[Set[str], Set[str]] | |
"""Returns a tuple of 2 sets of which paths to display to user | |
The first set contains paths that would be deleted. Files of a package | |
are not added and the top-level directory of the package has a '*' added | |
at the end - to signify that all it's contents are removed. | |
The second set contains files that would have been skipped in the above | |
folders. | |
""" | |
will_remove = set(paths) | |
will_skip = set() | |
# Determine folders and files | |
folders = set() | |
files = set() | |
for path in will_remove: | |
if path.endswith(".pyc"): | |
continue | |
if path.endswith("__init__.py") or ".dist-info" in path: | |
folders.add(os.path.dirname(path)) | |
files.add(path) | |
# probably this one https://github.com/python/mypy/issues/390 | |
_normcased_files = set(map(os.path.normcase, files)) # type: ignore | |
folders = compact(folders) | |
# This walks the tree using os.walk to not miss extra folders | |
# that might get added. | |
for folder in folders: | |
for dirpath, _, dirfiles in os.walk(folder): | |
for fname in dirfiles: | |
if fname.endswith(".pyc"): | |
continue | |
file_ = os.path.join(dirpath, fname) | |
if (os.path.isfile(file_) and | |
os.path.normcase(file_) not in _normcased_files): | |
# We are skipping this file. Add it to the set. | |
will_skip.add(file_) | |
will_remove = files | { | |
os.path.join(folder, "*") for folder in folders | |
} | |
return will_remove, will_skip | |
class StashedUninstallPathSet(object): | |
"""A set of file rename operations to stash files while | |
tentatively uninstalling them.""" | |
def __init__(self): | |
# type: () -> None | |
# Mapping from source file root to [Adjacent]TempDirectory | |
# for files under that directory. | |
self._save_dirs = {} # type: Dict[str, TempDirectory] | |
# (old path, new path) tuples for each move that may need | |
# to be undone. | |
self._moves = [] # type: List[Tuple[str, str]] | |
def _get_directory_stash(self, path): | |
# type: (str) -> str | |
"""Stashes a directory. | |
Directories are stashed adjacent to their original location if | |
possible, or else moved/copied into the user's temp dir.""" | |
try: | |
save_dir = AdjacentTempDirectory(path) # type: TempDirectory | |
except OSError: | |
save_dir = TempDirectory(kind="uninstall") | |
self._save_dirs[os.path.normcase(path)] = save_dir | |
return save_dir.path | |
def _get_file_stash(self, path): | |
# type: (str) -> str | |
"""Stashes a file. | |
If no root has been provided, one will be created for the directory | |
in the user's temp directory.""" | |
path = os.path.normcase(path) | |
head, old_head = os.path.dirname(path), None | |
save_dir = None | |
while head != old_head: | |
try: | |
save_dir = self._save_dirs[head] | |
break | |
except KeyError: | |
pass | |
head, old_head = os.path.dirname(head), head | |
else: | |
# Did not find any suitable root | |
head = os.path.dirname(path) | |
save_dir = TempDirectory(kind='uninstall') | |
self._save_dirs[head] = save_dir | |
relpath = os.path.relpath(path, head) | |
if relpath and relpath != os.path.curdir: | |
return os.path.join(save_dir.path, relpath) | |
return save_dir.path | |
def stash(self, path): | |
# type: (str) -> str | |
"""Stashes the directory or file and returns its new location. | |
Handle symlinks as files to avoid modifying the symlink targets. | |
""" | |
path_is_dir = os.path.isdir(path) and not os.path.islink(path) | |
if path_is_dir: | |
new_path = self._get_directory_stash(path) | |
else: | |
new_path = self._get_file_stash(path) | |
self._moves.append((path, new_path)) | |
if (path_is_dir and os.path.isdir(new_path)): | |
# If we're moving a directory, we need to | |
# remove the destination first or else it will be | |
# moved to inside the existing directory. | |
# We just created new_path ourselves, so it will | |
# be removable. | |
os.rmdir(new_path) | |
renames(path, new_path) | |
return new_path | |
def commit(self): | |
# type: () -> None | |
"""Commits the uninstall by removing stashed files.""" | |
for _, save_dir in self._save_dirs.items(): | |
save_dir.cleanup() | |
self._moves = [] | |
self._save_dirs = {} | |
def rollback(self): | |
# type: () -> None | |
"""Undoes the uninstall by moving stashed files back.""" | |
for p in self._moves: | |
logger.info("Moving to %s\n from %s", *p) | |
for new_path, path in self._moves: | |
try: | |
logger.debug('Replacing %s from %s', new_path, path) | |
if os.path.isfile(new_path) or os.path.islink(new_path): | |
os.unlink(new_path) | |
elif os.path.isdir(new_path): | |
rmtree(new_path) | |
renames(path, new_path) | |
except OSError as ex: | |
logger.error("Failed to restore %s", new_path) | |
logger.debug("Exception: %s", ex) | |
self.commit() | |
@property | |
def can_rollback(self): | |
# type: () -> bool | |
return bool(self._moves) | |
class UninstallPathSet(object): | |
"""A set of file paths to be removed in the uninstallation of a | |
requirement.""" | |
def __init__(self, dist): | |
# type: (Distribution) -> None | |
self.paths = set() # type: Set[str] | |
self._refuse = set() # type: Set[str] | |
self.pth = {} # type: Dict[str, UninstallPthEntries] | |
self.dist = dist | |
self._moved_paths = StashedUninstallPathSet() | |
def _permitted(self, path): | |
# type: (str) -> bool | |
""" | |
Return True if the given path is one we are permitted to | |
remove/modify, False otherwise. | |
""" | |
return is_local(path) | |
def add(self, path): | |
# type: (str) -> None | |
head, tail = os.path.split(path) | |
# we normalize the head to resolve parent directory symlinks, but not | |
# the tail, since we only want to uninstall symlinks, not their targets | |
path = os.path.join(normalize_path(head), os.path.normcase(tail)) | |
if not os.path.exists(path): | |
return | |
if self._permitted(path): | |
self.paths.add(path) | |
else: | |
self._refuse.add(path) | |
# __pycache__ files can show up after 'installed-files.txt' is created, | |
# due to imports | |
if os.path.splitext(path)[1] == '.py' and uses_pycache: | |
self.add(cache_from_source(path)) | |
def add_pth(self, pth_file, entry): | |
# type: (str, str) -> None | |
pth_file = normalize_path(pth_file) | |
if self._permitted(pth_file): | |
if pth_file not in self.pth: | |
self.pth[pth_file] = UninstallPthEntries(pth_file) | |
self.pth[pth_file].add(entry) | |
else: | |
self._refuse.add(pth_file) | |
def remove(self, auto_confirm=False, verbose=False): | |
# type: (bool, bool) -> None | |
"""Remove paths in ``self.paths`` with confirmation (unless | |
``auto_confirm`` is True).""" | |
if not self.paths: | |
logger.info( | |
"Can't uninstall '%s'. No files were found to uninstall.", | |
self.dist.project_name, | |
) | |
return | |
dist_name_version = ( | |
self.dist.project_name + "-" + self.dist.version | |
) | |
logger.info('Uninstalling %s:', dist_name_version) | |
with indent_log(): | |
if auto_confirm or self._allowed_to_proceed(verbose): | |
moved = self._moved_paths | |
for_rename = compress_for_rename(self.paths) | |
for path in sorted(compact(for_rename)): | |
moved.stash(path) | |
logger.debug('Removing file or directory %s', path) | |
for pth in self.pth.values(): | |
pth.remove() | |
logger.info('Successfully uninstalled %s', dist_name_version) | |
def _allowed_to_proceed(self, verbose): | |
# type: (bool) -> bool | |
"""Display which files would be deleted and prompt for confirmation | |
""" | |
def _display(msg, paths): | |
# type: (str, Iterable[str]) -> None | |
if not paths: | |
return | |
logger.info(msg) | |
with indent_log(): | |
for path in sorted(compact(paths)): | |
logger.info(path) | |
if not verbose: | |
will_remove, will_skip = compress_for_output_listing(self.paths) | |
else: | |
# In verbose mode, display all the files that are going to be | |
# deleted. | |
will_remove = set(self.paths) | |
will_skip = set() | |
_display('Would remove:', will_remove) | |
_display('Would not remove (might be manually added):', will_skip) | |
_display('Would not remove (outside of prefix):', self._refuse) | |
if verbose: | |
_display('Will actually move:', compress_for_rename(self.paths)) | |
return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' | |
def rollback(self): | |
# type: () -> None | |
"""Rollback the changes previously made by remove().""" | |
if not self._moved_paths.can_rollback: | |
logger.error( | |
"Can't roll back %s; was not uninstalled", | |
self.dist.project_name, | |
) | |
return | |
logger.info('Rolling back uninstall of %s', self.dist.project_name) | |
self._moved_paths.rollback() | |
for pth in self.pth.values(): | |
pth.rollback() | |
def commit(self): | |
# type: () -> None | |
"""Remove temporary save dir: rollback will no longer be possible.""" | |
self._moved_paths.commit() | |
@classmethod | |
def from_dist(cls, dist): | |
# type: (Distribution) -> UninstallPathSet | |
dist_path = normalize_path(dist.location) | |
if not dist_is_local(dist): | |
logger.info( | |
"Not uninstalling %s at %s, outside environment %s", | |
dist.key, | |
dist_path, | |
sys.prefix, | |
) | |
return cls(dist) | |
if dist_path in {p for p in {sysconfig.get_path("stdlib"), | |
sysconfig.get_path("platstdlib")} | |
if p}: | |
logger.info( | |
"Not uninstalling %s at %s, as it is in the standard library.", | |
dist.key, | |
dist_path, | |
) | |
return cls(dist) | |
paths_to_remove = cls(dist) | |
develop_egg_link = egg_link_path(dist) | |
develop_egg_link_egg_info = '{}.egg-info'.format( | |
pkg_resources.to_filename(dist.project_name)) | |
egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) | |
# Special case for distutils installed package | |
distutils_egg_info = getattr(dist._provider, 'path', None) | |
# Uninstall cases order do matter as in the case of 2 installs of the | |
# same package, pip needs to uninstall the currently detected version | |
if (egg_info_exists and dist.egg_info.endswith('.egg-info') and | |
not dist.egg_info.endswith(develop_egg_link_egg_info)): | |
# if dist.egg_info.endswith(develop_egg_link_egg_info), we | |
# are in fact in the develop_egg_link case | |
paths_to_remove.add(dist.egg_info) | |
if dist.has_metadata('installed-files.txt'): | |
for installed_file in dist.get_metadata( | |
'installed-files.txt').splitlines(): | |
path = os.path.normpath( | |
os.path.join(dist.egg_info, installed_file) | |
) | |
paths_to_remove.add(path) | |
# FIXME: need a test for this elif block | |
# occurs with --single-version-externally-managed/--record outside | |
# of pip | |
elif dist.has_metadata('top_level.txt'): | |
if dist.has_metadata('namespace_packages.txt'): | |
namespaces = dist.get_metadata('namespace_packages.txt') | |
else: | |
namespaces = [] | |
for top_level_pkg in [ | |
p for p | |
in dist.get_metadata('top_level.txt').splitlines() | |
if p and p not in namespaces]: | |
path = os.path.join(dist.location, top_level_pkg) | |
paths_to_remove.add(path) | |
paths_to_remove.add(path + '.py') | |
paths_to_remove.add(path + '.pyc') | |
paths_to_remove.add(path + '.pyo') | |
elif distutils_egg_info: | |
raise UninstallationError( | |
"Cannot uninstall {!r}. It is a distutils installed project " | |
"and thus we cannot accurately determine which files belong " | |
"to it which would lead to only a partial uninstall.".format( | |
dist.project_name, | |
) | |
) | |
elif dist.location.endswith('.egg'): | |
# package installed by easy_install | |
# We cannot match on dist.egg_name because it can slightly vary | |
# i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg | |
paths_to_remove.add(dist.location) | |
easy_install_egg = os.path.split(dist.location)[1] | |
easy_install_pth = os.path.join(os.path.dirname(dist.location), | |
'easy-install.pth') | |
paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) | |
elif egg_info_exists and dist.egg_info.endswith('.dist-info'): | |
for path in uninstallation_paths(dist): | |
paths_to_remove.add(path) | |
elif develop_egg_link: | |
# develop egg | |
with open(develop_egg_link, 'r') as fh: | |
link_pointer = os.path.normcase(fh.readline().strip()) | |
assert (link_pointer == dist.location), ( | |
'Egg-link %s does not match installed location of %s ' | |
'(at %s)' % (link_pointer, dist.project_name, dist.location) | |
) | |
paths_to_remove.add(develop_egg_link) | |
easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), | |
'easy-install.pth') | |
paths_to_remove.add_pth(easy_install_pth, dist.location) | |
else: | |
logger.debug( | |
'Not sure how to uninstall: %s - Check: %s', | |
dist, dist.location, | |
) | |
# find distutils scripts= scripts | |
if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): | |
for script in dist.metadata_listdir('scripts'): | |
if dist_in_usersite(dist): | |
bin_dir = bin_user | |
else: | |
bin_dir = bin_py | |
paths_to_remove.add(os.path.join(bin_dir, script)) | |
if WINDOWS: | |
paths_to_remove.add(os.path.join(bin_dir, script) + '.bat') | |
# find console_scripts | |
_scripts_to_remove = [] | |
console_scripts = dist.get_entry_map(group='console_scripts') | |
for name in console_scripts.keys(): | |
_scripts_to_remove.extend(_script_names(dist, name, False)) | |
# find gui_scripts | |
gui_scripts = dist.get_entry_map(group='gui_scripts') | |
for name in gui_scripts.keys(): | |
_scripts_to_remove.extend(_script_names(dist, name, True)) | |
for s in _scripts_to_remove: | |
paths_to_remove.add(s) | |
return paths_to_remove | |
class UninstallPthEntries(object): | |
def __init__(self, pth_file): | |
# type: (str) -> None | |
if not os.path.isfile(pth_file): | |
raise UninstallationError( | |
"Cannot remove entries from nonexistent file %s" % pth_file | |
) | |
self.file = pth_file | |
self.entries = set() # type: Set[str] | |
self._saved_lines = None # type: Optional[List[bytes]] | |
def add(self, entry): | |
# type: (str) -> None | |
entry = os.path.normcase(entry) | |
# On Windows, os.path.normcase converts the entry to use | |
# backslashes. This is correct for entries that describe absolute | |
# paths outside of site-packages, but all the others use forward | |
# slashes. | |
# os.path.splitdrive is used instead of os.path.isabs because isabs | |
# treats non-absolute paths with drive letter markings like c:foo\bar | |
# as absolute paths. It also does not recognize UNC paths if they don't | |
# have more than "\\sever\share". Valid examples: "\\server\share\" or | |
# "\\server\share\folder". Python 2.7.8+ support UNC in splitdrive. | |
if WINDOWS and not os.path.splitdrive(entry)[0]: | |
entry = entry.replace('\\', '/') | |
self.entries.add(entry) | |
def remove(self): | |
# type: () -> None | |
logger.debug('Removing pth entries from %s:', self.file) | |
with open(self.file, 'rb') as fh: | |
# windows uses '\r\n' with py3k, but uses '\n' with py2.x | |
lines = fh.readlines() | |
self._saved_lines = lines | |
if any(b'\r\n' in line for line in lines): | |
endline = '\r\n' | |
else: | |
endline = '\n' | |
# handle missing trailing newline | |
if lines and not lines[-1].endswith(endline.encode("utf-8")): | |
lines[-1] = lines[-1] + endline.encode("utf-8") | |
for entry in self.entries: | |
try: | |
logger.debug('Removing entry: %s', entry) | |
lines.remove((entry + endline).encode("utf-8")) | |
except ValueError: | |
pass | |
with open(self.file, 'wb') as fh: | |
fh.writelines(lines) | |
def rollback(self): | |
# type: () -> bool | |
if self._saved_lines is None: | |
logger.error( | |
'Cannot roll back changes to %s, none were made', self.file | |
) | |
return False | |
logger.debug('Rolling %s back to previous state', self.file) | |
with open(self.file, 'wb') as fh: | |
fh.writelines(self._saved_lines) | |
return True |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import datetime | |
import hashlib | |
import json | |
import logging | |
import os.path | |
import sys | |
from pip._vendor import pkg_resources | |
from pip._vendor.packaging import version as packaging_version | |
from pip._vendor.six import ensure_binary | |
from pip._internal.index.collector import LinkCollector | |
from pip._internal.index.package_finder import PackageFinder | |
from pip._internal.models.search_scope import SearchScope | |
from pip._internal.models.selection_prefs import SelectionPreferences | |
from pip._internal.utils.filesystem import ( | |
adjacent_tmp_file, | |
check_path_owner, | |
replace, | |
) | |
from pip._internal.utils.misc import ( | |
ensure_dir, | |
get_installed_version, | |
redact_auth_from_url, | |
) | |
from pip._internal.utils.packaging import get_installer | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
import optparse | |
from optparse import Values | |
from typing import Any, Dict, Text, Union | |
from pip._internal.network.session import PipSession | |
SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" | |
logger = logging.getLogger(__name__) | |
def make_link_collector( | |
session, # type: PipSession | |
options, # type: Values | |
suppress_no_index=False, # type: bool | |
): | |
# type: (...) -> LinkCollector | |
""" | |
:param session: The Session to use to make requests. | |
:param suppress_no_index: Whether to ignore the --no-index option | |
when constructing the SearchScope object. | |
""" | |
index_urls = [options.index_url] + options.extra_index_urls | |
if options.no_index and not suppress_no_index: | |
logger.debug( | |
'Ignoring indexes: %s', | |
','.join(redact_auth_from_url(url) for url in index_urls), | |
) | |
index_urls = [] | |
# Make sure find_links is a list before passing to create(). | |
find_links = options.find_links or [] | |
search_scope = SearchScope.create( | |
find_links=find_links, index_urls=index_urls, | |
) | |
link_collector = LinkCollector(session=session, search_scope=search_scope) | |
return link_collector | |
def _get_statefile_name(key): | |
# type: (Union[str, Text]) -> str | |
key_bytes = ensure_binary(key) | |
name = hashlib.sha224(key_bytes).hexdigest() | |
return name | |
class SelfCheckState(object): | |
def __init__(self, cache_dir): | |
# type: (str) -> None | |
self.state = {} # type: Dict[str, Any] | |
self.statefile_path = None | |
# Try to load the existing state | |
if cache_dir: | |
self.statefile_path = os.path.join( | |
cache_dir, "selfcheck", _get_statefile_name(self.key) | |
) | |
try: | |
with open(self.statefile_path) as statefile: | |
self.state = json.load(statefile) | |
except (IOError, ValueError, KeyError): | |
# Explicitly suppressing exceptions, since we don't want to | |
# error out if the cache file is invalid. | |
pass | |
@property | |
def key(self): | |
return sys.prefix | |
def save(self, pypi_version, current_time): | |
# type: (str, datetime.datetime) -> None | |
# If we do not have a path to cache in, don't bother saving. | |
if not self.statefile_path: | |
return | |
# Check to make sure that we own the directory | |
if not check_path_owner(os.path.dirname(self.statefile_path)): | |
return | |
# Now that we've ensured the directory is owned by this user, we'll go | |
# ahead and make sure that all our directories are created. | |
ensure_dir(os.path.dirname(self.statefile_path)) | |
state = { | |
# Include the key so it's easy to tell which pip wrote the | |
# file. | |
"key": self.key, | |
"last_check": current_time.strftime(SELFCHECK_DATE_FMT), | |
"pypi_version": pypi_version, | |
} | |
text = json.dumps(state, sort_keys=True, separators=(",", ":")) | |
with adjacent_tmp_file(self.statefile_path) as f: | |
f.write(ensure_binary(text)) | |
try: | |
# Since we have a prefix-specific state file, we can just | |
# overwrite whatever is there, no need to check. | |
replace(f.name, self.statefile_path) | |
except OSError: | |
# Best effort. | |
pass | |
def was_installed_by_pip(pkg): | |
# type: (str) -> bool | |
"""Checks whether pkg was installed by pip | |
This is used not to display the upgrade message when pip is in fact | |
installed by system package manager, such as dnf on Fedora. | |
""" | |
try: | |
dist = pkg_resources.get_distribution(pkg) | |
return "pip" == get_installer(dist) | |
except pkg_resources.DistributionNotFound: | |
return False | |
def pip_self_version_check(session, options): | |
# type: (PipSession, optparse.Values) -> None | |
"""Check for an update for pip. | |
Limit the frequency of checks to once per week. State is stored either in | |
the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix | |
of the pip script path. | |
""" | |
installed_version = get_installed_version("pip") | |
if not installed_version: | |
return | |
pip_version = packaging_version.parse(installed_version) | |
pypi_version = None | |
try: | |
state = SelfCheckState(cache_dir=options.cache_dir) | |
current_time = datetime.datetime.utcnow() | |
# Determine if we need to refresh the state | |
if "last_check" in state.state and "pypi_version" in state.state: | |
last_check = datetime.datetime.strptime( | |
state.state["last_check"], | |
SELFCHECK_DATE_FMT | |
) | |
if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60: | |
pypi_version = state.state["pypi_version"] | |
# Refresh the version if we need to or just see if we need to warn | |
if pypi_version is None: | |
# Lets use PackageFinder to see what the latest pip version is | |
link_collector = make_link_collector( | |
session, | |
options=options, | |
suppress_no_index=True, | |
) | |
# Pass allow_yanked=False so we don't suggest upgrading to a | |
# yanked version. | |
selection_prefs = SelectionPreferences( | |
allow_yanked=False, | |
allow_all_prereleases=False, # Explicitly set to False | |
) | |
finder = PackageFinder.create( | |
link_collector=link_collector, | |
selection_prefs=selection_prefs, | |
) | |
best_candidate = finder.find_best_candidate("pip").best_candidate | |
if best_candidate is None: | |
return | |
pypi_version = str(best_candidate.version) | |
# save that we've performed a check | |
state.save(pypi_version, current_time) | |
remote_version = packaging_version.parse(pypi_version) | |
local_version_is_older = ( | |
pip_version < remote_version and | |
pip_version.base_version != remote_version.base_version and | |
was_installed_by_pip('pip') | |
) | |
# Determine if our pypi_version is older | |
if not local_version_is_older: | |
return | |
# We cannot tell how the current pip is available in the current | |
# command context, so be pragmatic here and suggest the command | |
# that's always available. This does not accommodate spaces in | |
# `sys.executable`. | |
pip_cmd = "{} -m pip".format(sys.executable) | |
logger.warning( | |
"You are using pip version %s; however, version %s is " | |
"available.\nYou should consider upgrading via the " | |
"'%s install --upgrade pip' command.", | |
pip_version, pypi_version, pip_cmd | |
) | |
except Exception: | |
logger.debug( | |
"There was an error checking the latest version of pip", | |
exc_info=True, | |
) |
""" | |
This code wraps the vendored appdirs module to so the return values are | |
compatible for the current pip code base. | |
The intention is to rewrite current usages gradually, keeping the tests pass, | |
and eventually drop this after all usages are changed. | |
""" | |
from __future__ import absolute_import | |
import os | |
from pip._vendor import appdirs as _appdirs | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List | |
def user_cache_dir(appname): | |
# type: (str) -> str | |
return _appdirs.user_cache_dir(appname, appauthor=False) | |
def user_config_dir(appname, roaming=True): | |
# type: (str, bool) -> str | |
return _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming) | |
def user_data_dir(appname, roaming=False): | |
# type: (str, bool) -> str | |
return _appdirs.user_data_dir(appname, appauthor=False, roaming=roaming) | |
def site_config_dirs(appname): | |
# type: (str) -> List[str] | |
dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True) | |
if _appdirs.system not in ["win32", "darwin"]: | |
return dirval.split(os.pathsep) | |
return [dirval] |
"""Stuff that differs in different Python versions and platform | |
distributions.""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import, division | |
import codecs | |
import locale | |
import logging | |
import os | |
import shutil | |
import sys | |
from pip._vendor.six import PY2, text_type | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Text, Tuple, Union | |
try: | |
import ipaddress | |
except ImportError: | |
try: | |
from pip._vendor import ipaddress # type: ignore | |
except ImportError: | |
import ipaddr as ipaddress # type: ignore | |
ipaddress.ip_address = ipaddress.IPAddress # type: ignore | |
ipaddress.ip_network = ipaddress.IPNetwork # type: ignore | |
__all__ = [ | |
"ipaddress", "uses_pycache", "console_to_str", | |
"get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", | |
] | |
logger = logging.getLogger(__name__) | |
if PY2: | |
import imp | |
try: | |
cache_from_source = imp.cache_from_source # type: ignore | |
except AttributeError: | |
# does not use __pycache__ | |
cache_from_source = None | |
uses_pycache = cache_from_source is not None | |
else: | |
uses_pycache = True | |
from importlib.util import cache_from_source | |
if PY2: | |
# In Python 2.7, backslashreplace exists | |
# but does not support use for decoding. | |
# We implement our own replace handler for this | |
# situation, so that we can consistently use | |
# backslash replacement for all versions. | |
def backslashreplace_decode_fn(err): | |
raw_bytes = (err.object[i] for i in range(err.start, err.end)) | |
# Python 2 gave us characters - convert to numeric bytes | |
raw_bytes = (ord(b) for b in raw_bytes) | |
return u"".join(u"\\x%x" % c for c in raw_bytes), err.end | |
codecs.register_error( | |
"backslashreplace_decode", | |
backslashreplace_decode_fn, | |
) | |
backslashreplace_decode = "backslashreplace_decode" | |
else: | |
backslashreplace_decode = "backslashreplace" | |
def has_tls(): | |
# type: () -> bool | |
try: | |
import _ssl # noqa: F401 # ignore unused | |
return True | |
except ImportError: | |
pass | |
from pip._vendor.urllib3.util import IS_PYOPENSSL | |
return IS_PYOPENSSL | |
def str_to_display(data, desc=None): | |
# type: (Union[bytes, Text], Optional[str]) -> Text | |
""" | |
For display or logging purposes, convert a bytes object (or text) to | |
text (e.g. unicode in Python 2) safe for output. | |
:param desc: An optional phrase describing the input data, for use in | |
the log message if a warning is logged. Defaults to "Bytes object". | |
This function should never error out and so can take a best effort | |
approach. It is okay to be lossy if needed since the return value is | |
just for display. | |
We assume the data is in the locale preferred encoding. If it won't | |
decode properly, we warn the user but decode as best we can. | |
We also ensure that the output can be safely written to standard output | |
without encoding errors. | |
""" | |
if isinstance(data, text_type): | |
return data | |
# Otherwise, data is a bytes object (str in Python 2). | |
# First, get the encoding we assume. This is the preferred | |
# encoding for the locale, unless that is not found, or | |
# it is ASCII, in which case assume UTF-8 | |
encoding = locale.getpreferredencoding() | |
if (not encoding) or codecs.lookup(encoding).name == "ascii": | |
encoding = "utf-8" | |
# Now try to decode the data - if we fail, warn the user and | |
# decode with replacement. | |
try: | |
decoded_data = data.decode(encoding) | |
except UnicodeDecodeError: | |
if desc is None: | |
desc = 'Bytes object' | |
msg_format = '{} does not appear to be encoded as %s'.format(desc) | |
logger.warning(msg_format, encoding) | |
decoded_data = data.decode(encoding, errors=backslashreplace_decode) | |
# Make sure we can print the output, by encoding it to the output | |
# encoding with replacement of unencodable characters, and then | |
# decoding again. | |
# We use stderr's encoding because it's less likely to be | |
# redirected and if we don't find an encoding we skip this | |
# step (on the assumption that output is wrapped by something | |
# that won't fail). | |
# The double getattr is to deal with the possibility that we're | |
# being called in a situation where sys.__stderr__ doesn't exist, | |
# or doesn't have an encoding attribute. Neither of these cases | |
# should occur in normal pip use, but there's no harm in checking | |
# in case people use pip in (unsupported) unusual situations. | |
output_encoding = getattr(getattr(sys, "__stderr__", None), | |
"encoding", None) | |
if output_encoding: | |
output_encoded = decoded_data.encode( | |
output_encoding, | |
errors="backslashreplace" | |
) | |
decoded_data = output_encoded.decode(output_encoding) | |
return decoded_data | |
def console_to_str(data): | |
# type: (bytes) -> Text | |
"""Return a string, safe for output, of subprocess output. | |
""" | |
return str_to_display(data, desc='Subprocess output') | |
def get_path_uid(path): | |
# type: (str) -> int | |
""" | |
Return path's uid. | |
Does not follow symlinks: | |
https://github.com/pypa/pip/pull/935#discussion_r5307003 | |
Placed this function in compat due to differences on AIX and | |
Jython, that should eventually go away. | |
:raises OSError: When path is a symlink or can't be read. | |
""" | |
if hasattr(os, 'O_NOFOLLOW'): | |
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) | |
file_uid = os.fstat(fd).st_uid | |
os.close(fd) | |
else: # AIX and Jython | |
# WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW | |
if not os.path.islink(path): | |
# older versions of Jython don't have `os.fstat` | |
file_uid = os.stat(path).st_uid | |
else: | |
# raise OSError for parity with os.O_NOFOLLOW above | |
raise OSError( | |
"%s is a symlink; Will not return uid for symlinks" % path | |
) | |
return file_uid | |
def expanduser(path): | |
# type: (str) -> str | |
""" | |
Expand ~ and ~user constructions. | |
Includes a workaround for https://bugs.python.org/issue14768 | |
""" | |
expanded = os.path.expanduser(path) | |
if path.startswith('~/') and expanded.startswith('//'): | |
expanded = expanded[1:] | |
return expanded | |
# packages in the stdlib that may have installation metadata, but should not be | |
# considered 'installed'. this theoretically could be determined based on | |
# dist.location (py27:`sysconfig.get_paths()['stdlib']`, | |
# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may | |
# make this ineffective, so hard-coding | |
stdlib_pkgs = {"python", "wsgiref", "argparse"} | |
# windows detection, covers cpython and ironpython | |
WINDOWS = (sys.platform.startswith("win") or | |
(sys.platform == 'cli' and os.name == 'nt')) | |
def samefile(file1, file2): | |
# type: (str, str) -> bool | |
"""Provide an alternative for os.path.samefile on Windows/Python2""" | |
if hasattr(os.path, 'samefile'): | |
return os.path.samefile(file1, file2) | |
else: | |
path1 = os.path.normcase(os.path.abspath(file1)) | |
path2 = os.path.normcase(os.path.abspath(file2)) | |
return path1 == path2 | |
if hasattr(shutil, 'get_terminal_size'): | |
def get_terminal_size(): | |
# type: () -> Tuple[int, int] | |
""" | |
Returns a tuple (x, y) representing the width(x) and the height(y) | |
in characters of the terminal window. | |
""" | |
return tuple(shutil.get_terminal_size()) # type: ignore | |
else: | |
def get_terminal_size(): | |
# type: () -> Tuple[int, int] | |
""" | |
Returns a tuple (x, y) representing the width(x) and the height(y) | |
in characters of the terminal window. | |
""" | |
def ioctl_GWINSZ(fd): | |
try: | |
import fcntl | |
import termios | |
import struct | |
cr = struct.unpack_from( | |
'hh', | |
fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') | |
) | |
except Exception: | |
return None | |
if cr == (0, 0): | |
return None | |
return cr | |
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) | |
if not cr: | |
if sys.platform != "win32": | |
try: | |
fd = os.open(os.ctermid(), os.O_RDONLY) | |
cr = ioctl_GWINSZ(fd) | |
os.close(fd) | |
except Exception: | |
pass | |
if not cr: | |
cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) | |
return int(cr[1]), int(cr[0]) |
""" | |
A module that implements tooling to enable easy warnings about deprecations. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import warnings | |
from pip._vendor.packaging.version import parse | |
from pip import __version__ as current_version | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Optional | |
DEPRECATION_MSG_PREFIX = "DEPRECATION: " | |
class PipDeprecationWarning(Warning): | |
pass | |
_original_showwarning = None # type: Any | |
# Warnings <-> Logging Integration | |
def _showwarning(message, category, filename, lineno, file=None, line=None): | |
if file is not None: | |
if _original_showwarning is not None: | |
_original_showwarning( | |
message, category, filename, lineno, file, line, | |
) | |
elif issubclass(category, PipDeprecationWarning): | |
# We use a specially named logger which will handle all of the | |
# deprecation messages for pip. | |
logger = logging.getLogger("pip._internal.deprecations") | |
logger.warning(message) | |
else: | |
_original_showwarning( | |
message, category, filename, lineno, file, line, | |
) | |
def install_warning_logger(): | |
# type: () -> None | |
# Enable our Deprecation Warnings | |
warnings.simplefilter("default", PipDeprecationWarning, append=True) | |
global _original_showwarning | |
if _original_showwarning is None: | |
_original_showwarning = warnings.showwarning | |
warnings.showwarning = _showwarning | |
def deprecated(reason, replacement, gone_in, issue=None): | |
# type: (str, Optional[str], Optional[str], Optional[int]) -> None | |
"""Helper to deprecate existing functionality. | |
reason: | |
Textual reason shown to the user about why this functionality has | |
been deprecated. | |
replacement: | |
Textual suggestion shown to the user about what alternative | |
functionality they can use. | |
gone_in: | |
The version of pip does this functionality should get removed in. | |
Raises errors if pip's current version is greater than or equal to | |
this. | |
issue: | |
Issue number on the tracker that would serve as a useful place for | |
users to find related discussion and provide feedback. | |
Always pass replacement, gone_in and issue as keyword arguments for clarity | |
at the call site. | |
""" | |
# Construct a nice message. | |
# This is eagerly formatted as we want it to get logged as if someone | |
# typed this entire message out. | |
sentences = [ | |
(reason, DEPRECATION_MSG_PREFIX + "{}"), | |
(gone_in, "pip {} will remove support for this functionality."), | |
(replacement, "A possible replacement is {}."), | |
(issue, ( | |
"You can find discussion regarding this at " | |
"https://github.com/pypa/pip/issues/{}." | |
)), | |
] | |
message = " ".join( | |
template.format(val) for val, template in sentences if val is not None | |
) | |
# Raise as an error if it has to be removed. | |
if gone_in is not None and parse(current_version) >= parse(gone_in): | |
raise PipDeprecationWarning(message) | |
warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) |
from distutils.errors import DistutilsArgError | |
from distutils.fancy_getopt import FancyGetopt | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Dict, List | |
_options = [ | |
("exec-prefix=", None, ""), | |
("home=", None, ""), | |
("install-base=", None, ""), | |
("install-data=", None, ""), | |
("install-headers=", None, ""), | |
("install-lib=", None, ""), | |
("install-platlib=", None, ""), | |
("install-purelib=", None, ""), | |
("install-scripts=", None, ""), | |
("prefix=", None, ""), | |
("root=", None, ""), | |
("user", None, ""), | |
] | |
# typeshed doesn't permit Tuple[str, None, str], see python/typeshed#3469. | |
_distutils_getopt = FancyGetopt(_options) # type: ignore | |
def parse_distutils_args(args): | |
# type: (List[str]) -> Dict[str, str] | |
"""Parse provided arguments, returning an object that has the | |
matched arguments. | |
Any unknown arguments are ignored. | |
""" | |
result = {} | |
for arg in args: | |
try: | |
_, match = _distutils_getopt.getopt(args=[arg]) | |
except DistutilsArgError: | |
# We don't care about any other options, which here may be | |
# considered unrecognized since our option list is not | |
# exhaustive. | |
pass | |
else: | |
result.update(match.__dict__) | |
return result |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import codecs | |
import locale | |
import re | |
import sys | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Tuple, Text | |
BOMS = [ | |
(codecs.BOM_UTF8, 'utf-8'), | |
(codecs.BOM_UTF16, 'utf-16'), | |
(codecs.BOM_UTF16_BE, 'utf-16-be'), | |
(codecs.BOM_UTF16_LE, 'utf-16-le'), | |
(codecs.BOM_UTF32, 'utf-32'), | |
(codecs.BOM_UTF32_BE, 'utf-32-be'), | |
(codecs.BOM_UTF32_LE, 'utf-32-le'), | |
] # type: List[Tuple[bytes, Text]] | |
ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') | |
def auto_decode(data): | |
# type: (bytes) -> Text | |
"""Check a bytes string for a BOM to correctly detect the encoding | |
Fallback to locale.getpreferredencoding(False) like open() on Python3""" | |
for bom, encoding in BOMS: | |
if data.startswith(bom): | |
return data[len(bom):].decode(encoding) | |
# Lets check the first two lines as in PEP263 | |
for line in data.split(b'\n')[:2]: | |
if line[0:1] == b'#' and ENCODING_RE.search(line): | |
encoding = ENCODING_RE.search(line).groups()[0].decode('ascii') | |
return data.decode(encoding) | |
return data.decode( | |
locale.getpreferredencoding(False) or sys.getdefaultencoding(), | |
) |
import sys | |
from pip._internal.cli.main import main | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, List | |
def _wrapper(args=None): | |
# type: (Optional[List[str]]) -> int | |
"""Central wrapper for all old entrypoints. | |
Historically pip has had several entrypoints defined. Because of issues | |
arising from PATH, sys.path, multiple Pythons, their interactions, and most | |
of them having a pip installed, users suffer every time an entrypoint gets | |
moved. | |
To alleviate this pain, and provide a mechanism for warning users and | |
directing them to an appropriate place for help, we now define all of | |
our old entrypoints as wrappers for the current one. | |
""" | |
sys.stderr.write( | |
"WARNING: pip is being invoked by an old script wrapper. This will " | |
"fail in a future version of pip.\n" | |
"Please see https://github.com/pypa/pip/issues/5599 for advice on " | |
"fixing the underlying issue.\n" | |
"To avoid this problem you can invoke Python with '-m pip' instead of " | |
"running pip directly.\n" | |
) | |
return main(args) |
import errno | |
import os | |
import os.path | |
import random | |
import shutil | |
import stat | |
import sys | |
from contextlib import contextmanager | |
from tempfile import NamedTemporaryFile | |
# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is | |
# why we ignore the type on this import. | |
from pip._vendor.retrying import retry # type: ignore | |
from pip._vendor.six import PY2 | |
from pip._internal.utils.compat import get_path_uid | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast | |
if MYPY_CHECK_RUNNING: | |
from typing import BinaryIO, Iterator | |
class NamedTemporaryFileResult(BinaryIO): | |
@property | |
def file(self): | |
# type: () -> BinaryIO | |
pass | |
def check_path_owner(path): | |
# type: (str) -> bool | |
# If we don't have a way to check the effective uid of this process, then | |
# we'll just assume that we own the directory. | |
if sys.platform == "win32" or not hasattr(os, "geteuid"): | |
return True | |
assert os.path.isabs(path) | |
previous = None | |
while path != previous: | |
if os.path.lexists(path): | |
# Check if path is writable by current user. | |
if os.geteuid() == 0: | |
# Special handling for root user in order to handle properly | |
# cases where users use sudo without -H flag. | |
try: | |
path_uid = get_path_uid(path) | |
except OSError: | |
return False | |
return path_uid == 0 | |
else: | |
return os.access(path, os.W_OK) | |
else: | |
previous, path = path, os.path.dirname(path) | |
return False # assume we don't own the path | |
def copy2_fixed(src, dest): | |
# type: (str, str) -> None | |
"""Wrap shutil.copy2() but map errors copying socket files to | |
SpecialFileError as expected. | |
See also https://bugs.python.org/issue37700. | |
""" | |
try: | |
shutil.copy2(src, dest) | |
except (OSError, IOError): | |
for f in [src, dest]: | |
try: | |
is_socket_file = is_socket(f) | |
except OSError: | |
# An error has already occurred. Another error here is not | |
# a problem and we can ignore it. | |
pass | |
else: | |
if is_socket_file: | |
raise shutil.SpecialFileError("`%s` is a socket" % f) | |
raise | |
def is_socket(path): | |
# type: (str) -> bool | |
return stat.S_ISSOCK(os.lstat(path).st_mode) | |
@contextmanager | |
def adjacent_tmp_file(path): | |
# type: (str) -> Iterator[NamedTemporaryFileResult] | |
"""Given a path to a file, open a temp file next to it securely and ensure | |
it is written to disk after the context reaches its end. | |
""" | |
with NamedTemporaryFile( | |
delete=False, | |
dir=os.path.dirname(path), | |
prefix=os.path.basename(path), | |
suffix='.tmp', | |
) as f: | |
result = cast('NamedTemporaryFileResult', f) | |
try: | |
yield result | |
finally: | |
result.file.flush() | |
os.fsync(result.file.fileno()) | |
_replace_retry = retry(stop_max_delay=1000, wait_fixed=250) | |
if PY2: | |
@_replace_retry | |
def replace(src, dest): | |
# type: (str, str) -> None | |
try: | |
os.rename(src, dest) | |
except OSError: | |
os.remove(dest) | |
os.rename(src, dest) | |
else: | |
replace = _replace_retry(os.replace) | |
# test_writable_dir and _test_writable_dir_win are copied from Flit, | |
# with the author's agreement to also place them under pip's license. | |
def test_writable_dir(path): | |
# type: (str) -> bool | |
"""Check if a directory is writable. | |
Uses os.access() on POSIX, tries creating files on Windows. | |
""" | |
# If the directory doesn't exist, find the closest parent that does. | |
while not os.path.isdir(path): | |
parent = os.path.dirname(path) | |
if parent == path: | |
break # Should never get here, but infinite loops are bad | |
path = parent | |
if os.name == 'posix': | |
return os.access(path, os.W_OK) | |
return _test_writable_dir_win(path) | |
def _test_writable_dir_win(path): | |
# type: (str) -> bool | |
# os.access doesn't work on Windows: http://bugs.python.org/issue2528 | |
# and we can't use tempfile: http://bugs.python.org/issue22107 | |
basename = 'accesstest_deleteme_fishfingers_custard_' | |
alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789' | |
for i in range(10): | |
name = basename + ''.join(random.choice(alphabet) for _ in range(6)) | |
file = os.path.join(path, name) | |
try: | |
fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL) | |
except OSError as e: | |
if e.errno == errno.EEXIST: | |
continue | |
if e.errno == errno.EPERM: | |
# This could be because there's a directory with the same name. | |
# But it's highly unlikely there's a directory called that, | |
# so we'll assume it's because the parent dir is not writable. | |
return False | |
raise | |
else: | |
os.close(fd) | |
os.unlink(file) | |
return True | |
# This should never be reached | |
raise EnvironmentError( | |
'Unexpected condition testing for writable directory' | |
) |
"""Filetype information. | |
""" | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Tuple | |
WHEEL_EXTENSION = '.whl' | |
BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') # type: Tuple[str, ...] | |
XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', | |
'.tar.lz', '.tar.lzma') # type: Tuple[str, ...] | |
ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) # type: Tuple[str, ...] | |
TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') # type: Tuple[str, ...] | |
ARCHIVE_EXTENSIONS = ( | |
ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS | |
) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import os | |
import sys | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Tuple | |
def glibc_version_string(): | |
# type: () -> Optional[str] | |
"Returns glibc version string, or None if not using glibc." | |
return glibc_version_string_confstr() or glibc_version_string_ctypes() | |
def glibc_version_string_confstr(): | |
# type: () -> Optional[str] | |
"Primary implementation of glibc_version_string using os.confstr." | |
# os.confstr is quite a bit faster than ctypes.DLL. It's also less likely | |
# to be broken or missing. This strategy is used in the standard library | |
# platform module: | |
# https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183 | |
if sys.platform == "win32": | |
return None | |
try: | |
# os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": | |
_, version = os.confstr("CS_GNU_LIBC_VERSION").split() | |
except (AttributeError, OSError, ValueError): | |
# os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... | |
return None | |
return version | |
def glibc_version_string_ctypes(): | |
# type: () -> Optional[str] | |
"Fallback implementation of glibc_version_string using ctypes." | |
try: | |
import ctypes | |
except ImportError: | |
return None | |
# ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen | |
# manpage says, "If filename is NULL, then the returned handle is for the | |
# main program". This way we can let the linker do the work to figure out | |
# which libc our process is actually using. | |
process_namespace = ctypes.CDLL(None) | |
try: | |
gnu_get_libc_version = process_namespace.gnu_get_libc_version | |
except AttributeError: | |
# Symbol doesn't exist -> therefore, we are not linked to | |
# glibc. | |
return None | |
# Call gnu_get_libc_version, which returns a string like "2.5" | |
gnu_get_libc_version.restype = ctypes.c_char_p | |
version_str = gnu_get_libc_version() | |
# py2 / py3 compatibility: | |
if not isinstance(version_str, str): | |
version_str = version_str.decode("ascii") | |
return version_str | |
# platform.libc_ver regularly returns completely nonsensical glibc | |
# versions. E.g. on my computer, platform says: | |
# | |
# ~$ python2.7 -c 'import platform; print(platform.libc_ver())' | |
# ('glibc', '2.7') | |
# ~$ python3.5 -c 'import platform; print(platform.libc_ver())' | |
# ('glibc', '2.9') | |
# | |
# But the truth is: | |
# | |
# ~$ ldd --version | |
# ldd (Debian GLIBC 2.22-11) 2.22 | |
# | |
# This is unfortunate, because it means that the linehaul data on libc | |
# versions that was generated by pip 8.1.2 and earlier is useless and | |
# misleading. Solution: instead of using platform, use our code that actually | |
# works. | |
def libc_ver(): | |
# type: () -> Tuple[str, str] | |
"""Try to determine the glibc version | |
Returns a tuple of strings (lib, version) which default to empty strings | |
in case the lookup fails. | |
""" | |
glibc_version = glibc_version_string() | |
if glibc_version is None: | |
return ("", "") | |
else: | |
return ("glibc", glibc_version) |
from __future__ import absolute_import | |
import hashlib | |
from pip._vendor.six import iteritems, iterkeys, itervalues | |
from pip._internal.exceptions import ( | |
HashMismatch, | |
HashMissing, | |
InstallationError, | |
) | |
from pip._internal.utils.misc import read_chunks | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Dict, List, BinaryIO, NoReturn, Iterator | |
) | |
from pip._vendor.six import PY3 | |
if PY3: | |
from hashlib import _Hash | |
else: | |
from hashlib import _hash as _Hash | |
# The recommended hash algo of the moment. Change this whenever the state of | |
# the art changes; it won't hurt backward compatibility. | |
FAVORITE_HASH = 'sha256' | |
# Names of hashlib algorithms allowed by the --hash option and ``pip hash`` | |
# Currently, those are the ones at least as collision-resistant as sha256. | |
STRONG_HASHES = ['sha256', 'sha384', 'sha512'] | |
class Hashes(object): | |
"""A wrapper that builds multiple hashes at once and checks them against | |
known-good values | |
""" | |
def __init__(self, hashes=None): | |
# type: (Dict[str, List[str]]) -> None | |
""" | |
:param hashes: A dict of algorithm names pointing to lists of allowed | |
hex digests | |
""" | |
self._allowed = {} if hashes is None else hashes | |
@property | |
def digest_count(self): | |
# type: () -> int | |
return sum(len(digests) for digests in self._allowed.values()) | |
def is_hash_allowed( | |
self, | |
hash_name, # type: str | |
hex_digest, # type: str | |
): | |
# type: (...) -> bool | |
"""Return whether the given hex digest is allowed.""" | |
return hex_digest in self._allowed.get(hash_name, []) | |
def check_against_chunks(self, chunks): | |
# type: (Iterator[bytes]) -> None | |
"""Check good hashes against ones built from iterable of chunks of | |
data. | |
Raise HashMismatch if none match. | |
""" | |
gots = {} | |
for hash_name in iterkeys(self._allowed): | |
try: | |
gots[hash_name] = hashlib.new(hash_name) | |
except (ValueError, TypeError): | |
raise InstallationError('Unknown hash name: %s' % hash_name) | |
for chunk in chunks: | |
for hash in itervalues(gots): | |
hash.update(chunk) | |
for hash_name, got in iteritems(gots): | |
if got.hexdigest() in self._allowed[hash_name]: | |
return | |
self._raise(gots) | |
def _raise(self, gots): | |
# type: (Dict[str, _Hash]) -> NoReturn | |
raise HashMismatch(self._allowed, gots) | |
def check_against_file(self, file): | |
# type: (BinaryIO) -> None | |
"""Check good hashes against a file-like object | |
Raise HashMismatch if none match. | |
""" | |
return self.check_against_chunks(read_chunks(file)) | |
def check_against_path(self, path): | |
# type: (str) -> None | |
with open(path, 'rb') as file: | |
return self.check_against_file(file) | |
def __nonzero__(self): | |
# type: () -> bool | |
"""Return whether I know any known-good hashes.""" | |
return bool(self._allowed) | |
def __bool__(self): | |
# type: () -> bool | |
return self.__nonzero__() | |
class MissingHashes(Hashes): | |
"""A workalike for Hashes used when we're missing a hash for a requirement | |
It computes the actual hash of the requirement and raises a HashMissing | |
exception showing it to the user. | |
""" | |
def __init__(self): | |
# type: () -> None | |
"""Don't offer the ``hashes`` kwarg.""" | |
# Pass our favorite hash in to generate a "gotten hash". With the | |
# empty list, it will never match, so an error will always raise. | |
super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) | |
def _raise(self, gots): | |
# type: (Dict[str, _Hash]) -> NoReturn | |
raise HashMissing(gots[FAVORITE_HASH].hexdigest()) |
"""A helper module that injects SecureTransport, on import. | |
The import should be done as early as possible, to ensure all requests and | |
sessions (or whatever) are created after injecting SecureTransport. | |
Note that we only do the injection on macOS, when the linked OpenSSL is too | |
old to handle TLSv1.2. | |
""" | |
import sys | |
def inject_securetransport(): | |
# type: () -> None | |
# Only relevant on macOS | |
if sys.platform != "darwin": | |
return | |
try: | |
import ssl | |
except ImportError: | |
return | |
# Checks for OpenSSL 1.0.1 | |
if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100f: | |
return | |
try: | |
from pip._vendor.urllib3.contrib import securetransport | |
except (ImportError, OSError): | |
return | |
securetransport.inject_into_urllib3() | |
inject_securetransport() |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import contextlib | |
import errno | |
import logging | |
import logging.handlers | |
import os | |
import sys | |
from logging import Filter, getLogger | |
from pip._vendor.six import PY2 | |
from pip._internal.utils.compat import WINDOWS | |
from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX | |
from pip._internal.utils.misc import ensure_dir | |
try: | |
import threading | |
except ImportError: | |
import dummy_threading as threading # type: ignore | |
try: | |
# Use "import as" and set colorama in the else clause to avoid mypy | |
# errors and get the following correct revealed type for colorama: | |
# `Union[_importlib_modulespec.ModuleType, None]` | |
# Otherwise, we get an error like the following in the except block: | |
# > Incompatible types in assignment (expression has type "None", | |
# variable has type Module) | |
# TODO: eliminate the need to use "import as" once mypy addresses some | |
# of its issues with conditional imports. Here is an umbrella issue: | |
# https://github.com/python/mypy/issues/1297 | |
from pip._vendor import colorama as _colorama | |
# Lots of different errors can come from this, including SystemError and | |
# ImportError. | |
except Exception: | |
colorama = None | |
else: | |
# Import Fore explicitly rather than accessing below as colorama.Fore | |
# to avoid the following error running mypy: | |
# > Module has no attribute "Fore" | |
# TODO: eliminate the need to import Fore once mypy addresses some of its | |
# issues with conditional imports. This particular case could be an | |
# instance of the following issue (but also see the umbrella issue above): | |
# https://github.com/python/mypy/issues/3500 | |
from pip._vendor.colorama import Fore | |
colorama = _colorama | |
_log_state = threading.local() | |
_log_state.indentation = 0 | |
subprocess_logger = getLogger('pip.subprocessor') | |
class BrokenStdoutLoggingError(Exception): | |
""" | |
Raised if BrokenPipeError occurs for the stdout stream while logging. | |
""" | |
pass | |
# BrokenPipeError does not exist in Python 2 and, in addition, manifests | |
# differently in Windows and non-Windows. | |
if WINDOWS: | |
# In Windows, a broken pipe can show up as EINVAL rather than EPIPE: | |
# https://bugs.python.org/issue19612 | |
# https://bugs.python.org/issue30418 | |
if PY2: | |
def _is_broken_pipe_error(exc_class, exc): | |
"""See the docstring for non-Windows Python 3 below.""" | |
return (exc_class is IOError and | |
exc.errno in (errno.EINVAL, errno.EPIPE)) | |
else: | |
# In Windows, a broken pipe IOError became OSError in Python 3. | |
def _is_broken_pipe_error(exc_class, exc): | |
"""See the docstring for non-Windows Python 3 below.""" | |
return ((exc_class is BrokenPipeError) or # noqa: F821 | |
(exc_class is OSError and | |
exc.errno in (errno.EINVAL, errno.EPIPE))) | |
elif PY2: | |
def _is_broken_pipe_error(exc_class, exc): | |
"""See the docstring for non-Windows Python 3 below.""" | |
return (exc_class is IOError and exc.errno == errno.EPIPE) | |
else: | |
# Then we are in the non-Windows Python 3 case. | |
def _is_broken_pipe_error(exc_class, exc): | |
""" | |
Return whether an exception is a broken pipe error. | |
Args: | |
exc_class: an exception class. | |
exc: an exception instance. | |
""" | |
return (exc_class is BrokenPipeError) # noqa: F821 | |
@contextlib.contextmanager | |
def indent_log(num=2): | |
""" | |
A context manager which will cause the log output to be indented for any | |
log messages emitted inside it. | |
""" | |
_log_state.indentation += num | |
try: | |
yield | |
finally: | |
_log_state.indentation -= num | |
def get_indentation(): | |
return getattr(_log_state, 'indentation', 0) | |
class IndentingFormatter(logging.Formatter): | |
def __init__(self, *args, **kwargs): | |
""" | |
A logging.Formatter that obeys the indent_log() context manager. | |
:param add_timestamp: A bool indicating output lines should be prefixed | |
with their record's timestamp. | |
""" | |
self.add_timestamp = kwargs.pop("add_timestamp", False) | |
super(IndentingFormatter, self).__init__(*args, **kwargs) | |
def get_message_start(self, formatted, levelno): | |
""" | |
Return the start of the formatted log message (not counting the | |
prefix to add to each line). | |
""" | |
if levelno < logging.WARNING: | |
return '' | |
if formatted.startswith(DEPRECATION_MSG_PREFIX): | |
# Then the message already has a prefix. We don't want it to | |
# look like "WARNING: DEPRECATION: ...." | |
return '' | |
if levelno < logging.ERROR: | |
return 'WARNING: ' | |
return 'ERROR: ' | |
def format(self, record): | |
""" | |
Calls the standard formatter, but will indent all of the log message | |
lines by our current indentation level. | |
""" | |
formatted = super(IndentingFormatter, self).format(record) | |
message_start = self.get_message_start(formatted, record.levelno) | |
formatted = message_start + formatted | |
prefix = '' | |
if self.add_timestamp: | |
# TODO: Use Formatter.default_time_format after dropping PY2. | |
t = self.formatTime(record, "%Y-%m-%dT%H:%M:%S") | |
prefix = '%s,%03d ' % (t, record.msecs) | |
prefix += " " * get_indentation() | |
formatted = "".join([ | |
prefix + line | |
for line in formatted.splitlines(True) | |
]) | |
return formatted | |
def _color_wrap(*colors): | |
def wrapped(inp): | |
return "".join(list(colors) + [inp, colorama.Style.RESET_ALL]) | |
return wrapped | |
class ColorizedStreamHandler(logging.StreamHandler): | |
# Don't build up a list of colors if we don't have colorama | |
if colorama: | |
COLORS = [ | |
# This needs to be in order from highest logging level to lowest. | |
(logging.ERROR, _color_wrap(Fore.RED)), | |
(logging.WARNING, _color_wrap(Fore.YELLOW)), | |
] | |
else: | |
COLORS = [] | |
def __init__(self, stream=None, no_color=None): | |
logging.StreamHandler.__init__(self, stream) | |
self._no_color = no_color | |
if WINDOWS and colorama: | |
self.stream = colorama.AnsiToWin32(self.stream) | |
def _using_stdout(self): | |
""" | |
Return whether the handler is using sys.stdout. | |
""" | |
if WINDOWS and colorama: | |
# Then self.stream is an AnsiToWin32 object. | |
return self.stream.wrapped is sys.stdout | |
return self.stream is sys.stdout | |
def should_color(self): | |
# Don't colorize things if we do not have colorama or if told not to | |
if not colorama or self._no_color: | |
return False | |
real_stream = ( | |
self.stream if not isinstance(self.stream, colorama.AnsiToWin32) | |
else self.stream.wrapped | |
) | |
# If the stream is a tty we should color it | |
if hasattr(real_stream, "isatty") and real_stream.isatty(): | |
return True | |
# If we have an ANSI term we should color it | |
if os.environ.get("TERM") == "ANSI": | |
return True | |
# If anything else we should not color it | |
return False | |
def format(self, record): | |
msg = logging.StreamHandler.format(self, record) | |
if self.should_color(): | |
for level, color in self.COLORS: | |
if record.levelno >= level: | |
msg = color(msg) | |
break | |
return msg | |
# The logging module says handleError() can be customized. | |
def handleError(self, record): | |
exc_class, exc = sys.exc_info()[:2] | |
# If a broken pipe occurred while calling write() or flush() on the | |
# stdout stream in logging's Handler.emit(), then raise our special | |
# exception so we can handle it in main() instead of logging the | |
# broken pipe error and continuing. | |
if (exc_class and self._using_stdout() and | |
_is_broken_pipe_error(exc_class, exc)): | |
raise BrokenStdoutLoggingError() | |
return super(ColorizedStreamHandler, self).handleError(record) | |
class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): | |
def _open(self): | |
ensure_dir(os.path.dirname(self.baseFilename)) | |
return logging.handlers.RotatingFileHandler._open(self) | |
class MaxLevelFilter(Filter): | |
def __init__(self, level): | |
self.level = level | |
def filter(self, record): | |
return record.levelno < self.level | |
class ExcludeLoggerFilter(Filter): | |
""" | |
A logging Filter that excludes records from a logger (or its children). | |
""" | |
def filter(self, record): | |
# The base Filter class allows only records from a logger (or its | |
# children). | |
return not super(ExcludeLoggerFilter, self).filter(record) | |
def setup_logging(verbosity, no_color, user_log_file): | |
"""Configures and sets up all of the logging | |
Returns the requested logging level, as its integer value. | |
""" | |
# Determine the level to be logging at. | |
if verbosity >= 1: | |
level = "DEBUG" | |
elif verbosity == -1: | |
level = "WARNING" | |
elif verbosity == -2: | |
level = "ERROR" | |
elif verbosity <= -3: | |
level = "CRITICAL" | |
else: | |
level = "INFO" | |
level_number = getattr(logging, level) | |
# The "root" logger should match the "console" level *unless* we also need | |
# to log to a user log file. | |
include_user_log = user_log_file is not None | |
if include_user_log: | |
additional_log_file = user_log_file | |
root_level = "DEBUG" | |
else: | |
additional_log_file = "/dev/null" | |
root_level = level | |
# Disable any logging besides WARNING unless we have DEBUG level logging | |
# enabled for vendored libraries. | |
vendored_log_level = "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" | |
# Shorthands for clarity | |
log_streams = { | |
"stdout": "ext://sys.stdout", | |
"stderr": "ext://sys.stderr", | |
} | |
handler_classes = { | |
"stream": "pip._internal.utils.logging.ColorizedStreamHandler", | |
"file": "pip._internal.utils.logging.BetterRotatingFileHandler", | |
} | |
handlers = ["console", "console_errors", "console_subprocess"] + ( | |
["user_log"] if include_user_log else [] | |
) | |
logging.config.dictConfig({ | |
"version": 1, | |
"disable_existing_loggers": False, | |
"filters": { | |
"exclude_warnings": { | |
"()": "pip._internal.utils.logging.MaxLevelFilter", | |
"level": logging.WARNING, | |
}, | |
"restrict_to_subprocess": { | |
"()": "logging.Filter", | |
"name": subprocess_logger.name, | |
}, | |
"exclude_subprocess": { | |
"()": "pip._internal.utils.logging.ExcludeLoggerFilter", | |
"name": subprocess_logger.name, | |
}, | |
}, | |
"formatters": { | |
"indent": { | |
"()": IndentingFormatter, | |
"format": "%(message)s", | |
}, | |
"indent_with_timestamp": { | |
"()": IndentingFormatter, | |
"format": "%(message)s", | |
"add_timestamp": True, | |
}, | |
}, | |
"handlers": { | |
"console": { | |
"level": level, | |
"class": handler_classes["stream"], | |
"no_color": no_color, | |
"stream": log_streams["stdout"], | |
"filters": ["exclude_subprocess", "exclude_warnings"], | |
"formatter": "indent", | |
}, | |
"console_errors": { | |
"level": "WARNING", | |
"class": handler_classes["stream"], | |
"no_color": no_color, | |
"stream": log_streams["stderr"], | |
"filters": ["exclude_subprocess"], | |
"formatter": "indent", | |
}, | |
# A handler responsible for logging to the console messages | |
# from the "subprocessor" logger. | |
"console_subprocess": { | |
"level": level, | |
"class": handler_classes["stream"], | |
"no_color": no_color, | |
"stream": log_streams["stderr"], | |
"filters": ["restrict_to_subprocess"], | |
"formatter": "indent", | |
}, | |
"user_log": { | |
"level": "DEBUG", | |
"class": handler_classes["file"], | |
"filename": additional_log_file, | |
"delay": True, | |
"formatter": "indent_with_timestamp", | |
}, | |
}, | |
"root": { | |
"level": root_level, | |
"handlers": handlers, | |
}, | |
"loggers": { | |
"pip._vendor": { | |
"level": vendored_log_level | |
} | |
}, | |
}) | |
return level_number |
import os.path | |
DELETE_MARKER_MESSAGE = '''\ | |
This file is placed here by pip to indicate the source was put | |
here by pip. | |
Once this package is successfully installed this source code will be | |
deleted (unless you remove this file). | |
''' | |
PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' | |
def has_delete_marker_file(directory): | |
# type: (str) -> bool | |
return os.path.exists(os.path.join(directory, PIP_DELETE_MARKER_FILENAME)) | |
def write_delete_marker_file(directory): | |
# type: (str) -> None | |
""" | |
Write the pip delete marker file into this directory. | |
""" | |
filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME) | |
with open(filepath, 'w') as marker_fp: | |
marker_fp.write(DELETE_MARKER_MESSAGE) |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import contextlib | |
import errno | |
import getpass | |
import hashlib | |
import io | |
import logging | |
import os | |
import posixpath | |
import shutil | |
import stat | |
import sys | |
from collections import deque | |
from pip._vendor import pkg_resources | |
# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is | |
# why we ignore the type on this import. | |
from pip._vendor.retrying import retry # type: ignore | |
from pip._vendor.six import PY2, text_type | |
from pip._vendor.six.moves import input | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote | |
from pip import __version__ | |
from pip._internal.exceptions import CommandError | |
from pip._internal.locations import ( | |
get_major_minor_version, | |
site_packages, | |
user_site, | |
) | |
from pip._internal.utils.compat import ( | |
WINDOWS, | |
expanduser, | |
stdlib_pkgs, | |
str_to_display, | |
) | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast | |
from pip._internal.utils.virtualenv import ( | |
running_under_virtualenv, | |
virtualenv_no_global, | |
) | |
if PY2: | |
from io import BytesIO as StringIO | |
else: | |
from io import StringIO | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, AnyStr, Container, Iterable, List, Optional, Text, | |
Tuple, Union, | |
) | |
from pip._vendor.pkg_resources import Distribution | |
VersionInfo = Tuple[int, int, int] | |
__all__ = ['rmtree', 'display_path', 'backup_dir', | |
'ask', 'splitext', | |
'format_size', 'is_installable_dir', | |
'normalize_path', | |
'renames', 'get_prog', | |
'captured_stdout', 'ensure_dir', | |
'get_installed_version', 'remove_auth_from_url'] | |
logger = logging.getLogger(__name__) | |
def get_pip_version(): | |
# type: () -> str | |
pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..") | |
pip_pkg_dir = os.path.abspath(pip_pkg_dir) | |
return ( | |
'pip {} from {} (python {})'.format( | |
__version__, pip_pkg_dir, get_major_minor_version(), | |
) | |
) | |
def normalize_version_info(py_version_info): | |
# type: (Tuple[int, ...]) -> Tuple[int, int, int] | |
""" | |
Convert a tuple of ints representing a Python version to one of length | |
three. | |
:param py_version_info: a tuple of ints representing a Python version, | |
or None to specify no version. The tuple can have any length. | |
:return: a tuple of length three if `py_version_info` is non-None. | |
Otherwise, return `py_version_info` unchanged (i.e. None). | |
""" | |
if len(py_version_info) < 3: | |
py_version_info += (3 - len(py_version_info)) * (0,) | |
elif len(py_version_info) > 3: | |
py_version_info = py_version_info[:3] | |
return cast('VersionInfo', py_version_info) | |
def ensure_dir(path): | |
# type: (AnyStr) -> None | |
"""os.path.makedirs without EEXIST.""" | |
try: | |
os.makedirs(path) | |
except OSError as e: | |
# Windows can raise spurious ENOTEMPTY errors. See #6426. | |
if e.errno != errno.EEXIST and e.errno != errno.ENOTEMPTY: | |
raise | |
def get_prog(): | |
# type: () -> str | |
try: | |
prog = os.path.basename(sys.argv[0]) | |
if prog in ('__main__.py', '-c'): | |
return "%s -m pip" % sys.executable | |
else: | |
return prog | |
except (AttributeError, TypeError, IndexError): | |
pass | |
return 'pip' | |
# Retry every half second for up to 3 seconds | |
@retry(stop_max_delay=3000, wait_fixed=500) | |
def rmtree(dir, ignore_errors=False): | |
# type: (str, bool) -> None | |
shutil.rmtree(dir, ignore_errors=ignore_errors, | |
onerror=rmtree_errorhandler) | |
def rmtree_errorhandler(func, path, exc_info): | |
"""On Windows, the files in .svn are read-only, so when rmtree() tries to | |
remove them, an exception is thrown. We catch that here, remove the | |
read-only attribute, and hopefully continue without problems.""" | |
try: | |
has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE) | |
except (IOError, OSError): | |
# it's equivalent to os.path.exists | |
return | |
if has_attr_readonly: | |
# convert to read/write | |
os.chmod(path, stat.S_IWRITE) | |
# use the original function to repeat the operation | |
func(path) | |
return | |
else: | |
raise | |
def path_to_display(path): | |
# type: (Optional[Union[str, Text]]) -> Optional[Text] | |
""" | |
Convert a bytes (or text) path to text (unicode in Python 2) for display | |
and logging purposes. | |
This function should never error out. Also, this function is mainly needed | |
for Python 2 since in Python 3 str paths are already text. | |
""" | |
if path is None: | |
return None | |
if isinstance(path, text_type): | |
return path | |
# Otherwise, path is a bytes object (str in Python 2). | |
try: | |
display_path = path.decode(sys.getfilesystemencoding(), 'strict') | |
except UnicodeDecodeError: | |
# Include the full bytes to make troubleshooting easier, even though | |
# it may not be very human readable. | |
if PY2: | |
# Convert the bytes to a readable str representation using | |
# repr(), and then convert the str to unicode. | |
# Also, we add the prefix "b" to the repr() return value both | |
# to make the Python 2 output look like the Python 3 output, and | |
# to signal to the user that this is a bytes representation. | |
display_path = str_to_display('b{!r}'.format(path)) | |
else: | |
# Silence the "F821 undefined name 'ascii'" flake8 error since | |
# in Python 3 ascii() is a built-in. | |
display_path = ascii(path) # noqa: F821 | |
return display_path | |
def display_path(path): | |
# type: (Union[str, Text]) -> str | |
"""Gives the display value for a given path, making it relative to cwd | |
if possible.""" | |
path = os.path.normcase(os.path.abspath(path)) | |
if sys.version_info[0] == 2: | |
path = path.decode(sys.getfilesystemencoding(), 'replace') | |
path = path.encode(sys.getdefaultencoding(), 'replace') | |
if path.startswith(os.getcwd() + os.path.sep): | |
path = '.' + path[len(os.getcwd()):] | |
return path | |
def backup_dir(dir, ext='.bak'): | |
# type: (str, str) -> str | |
"""Figure out the name of a directory to back up the given dir to | |
(adding .bak, .bak2, etc)""" | |
n = 1 | |
extension = ext | |
while os.path.exists(dir + extension): | |
n += 1 | |
extension = ext + str(n) | |
return dir + extension | |
def ask_path_exists(message, options): | |
# type: (str, Iterable[str]) -> str | |
for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): | |
if action in options: | |
return action | |
return ask(message, options) | |
def _check_no_input(message): | |
# type: (str) -> None | |
"""Raise an error if no input is allowed.""" | |
if os.environ.get('PIP_NO_INPUT'): | |
raise Exception( | |
'No input was expected ($PIP_NO_INPUT set); question: %s' % | |
message | |
) | |
def ask(message, options): | |
# type: (str, Iterable[str]) -> str | |
"""Ask the message interactively, with the given possible responses""" | |
while 1: | |
_check_no_input(message) | |
response = input(message) | |
response = response.strip().lower() | |
if response not in options: | |
print( | |
'Your response (%r) was not one of the expected responses: ' | |
'%s' % (response, ', '.join(options)) | |
) | |
else: | |
return response | |
def ask_input(message): | |
# type: (str) -> str | |
"""Ask for input interactively.""" | |
_check_no_input(message) | |
return input(message) | |
def ask_password(message): | |
# type: (str) -> str | |
"""Ask for a password interactively.""" | |
_check_no_input(message) | |
return getpass.getpass(message) | |
def format_size(bytes): | |
# type: (float) -> str | |
if bytes > 1000 * 1000: | |
return '%.1f MB' % (bytes / 1000.0 / 1000) | |
elif bytes > 10 * 1000: | |
return '%i kB' % (bytes / 1000) | |
elif bytes > 1000: | |
return '%.1f kB' % (bytes / 1000.0) | |
else: | |
return '%i bytes' % bytes | |
def is_installable_dir(path): | |
# type: (str) -> bool | |
"""Is path is a directory containing setup.py or pyproject.toml? | |
""" | |
if not os.path.isdir(path): | |
return False | |
setup_py = os.path.join(path, 'setup.py') | |
if os.path.isfile(setup_py): | |
return True | |
pyproject_toml = os.path.join(path, 'pyproject.toml') | |
if os.path.isfile(pyproject_toml): | |
return True | |
return False | |
def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): | |
"""Yield pieces of data from a file-like object until EOF.""" | |
while True: | |
chunk = file.read(size) | |
if not chunk: | |
break | |
yield chunk | |
def normalize_path(path, resolve_symlinks=True): | |
# type: (str, bool) -> str | |
""" | |
Convert a path to its canonical, case-normalized, absolute version. | |
""" | |
path = expanduser(path) | |
if resolve_symlinks: | |
path = os.path.realpath(path) | |
else: | |
path = os.path.abspath(path) | |
return os.path.normcase(path) | |
def splitext(path): | |
# type: (str) -> Tuple[str, str] | |
"""Like os.path.splitext, but take off .tar too""" | |
base, ext = posixpath.splitext(path) | |
if base.lower().endswith('.tar'): | |
ext = base[-4:] + ext | |
base = base[:-4] | |
return base, ext | |
def renames(old, new): | |
# type: (str, str) -> None | |
"""Like os.renames(), but handles renaming across devices.""" | |
# Implementation borrowed from os.renames(). | |
head, tail = os.path.split(new) | |
if head and tail and not os.path.exists(head): | |
os.makedirs(head) | |
shutil.move(old, new) | |
head, tail = os.path.split(old) | |
if head and tail: | |
try: | |
os.removedirs(head) | |
except OSError: | |
pass | |
def is_local(path): | |
# type: (str) -> bool | |
""" | |
Return True if this is a path pip is allowed to modify. | |
If we're in a virtualenv, sys.prefix points to the virtualenv's | |
prefix; only sys.prefix is considered local. | |
If we're not in a virtualenv, in general we can modify anything. | |
However, if the OS vendor has configured distutils to install | |
somewhere other than sys.prefix (which could be a subdirectory of | |
sys.prefix, e.g. /usr/local), we consider sys.prefix itself nonlocal | |
and the domain of the OS vendor. (In other words, everything _other | |
than_ sys.prefix is considered local.) | |
Caution: this function assumes the head of path has been normalized | |
with normalize_path. | |
""" | |
path = normalize_path(path) | |
prefix = normalize_path(sys.prefix) | |
if running_under_virtualenv(): | |
return path.startswith(normalize_path(sys.prefix)) | |
else: | |
from pip._internal.locations import distutils_scheme | |
if path.startswith(prefix): | |
for local_path in distutils_scheme("").values(): | |
if path.startswith(normalize_path(local_path)): | |
return True | |
return False | |
else: | |
return True | |
def dist_is_local(dist): | |
# type: (Distribution) -> bool | |
""" | |
Return True if given Distribution object is installed somewhere pip | |
is allowed to modify. | |
""" | |
return is_local(dist_location(dist)) | |
def dist_in_usersite(dist): | |
# type: (Distribution) -> bool | |
""" | |
Return True if given Distribution is installed in user site. | |
""" | |
return dist_location(dist).startswith(normalize_path(user_site)) | |
def dist_in_site_packages(dist): | |
# type: (Distribution) -> bool | |
""" | |
Return True if given Distribution is installed in | |
sysconfig.get_python_lib(). | |
""" | |
return dist_location(dist).startswith(normalize_path(site_packages)) | |
def dist_is_editable(dist): | |
# type: (Distribution) -> bool | |
""" | |
Return True if given Distribution is an editable install. | |
""" | |
for path_item in sys.path: | |
egg_link = os.path.join(path_item, dist.project_name + '.egg-link') | |
if os.path.isfile(egg_link): | |
return True | |
return False | |
def get_installed_distributions( | |
local_only=True, # type: bool | |
skip=stdlib_pkgs, # type: Container[str] | |
include_editables=True, # type: bool | |
editables_only=False, # type: bool | |
user_only=False, # type: bool | |
paths=None # type: Optional[List[str]] | |
): | |
# type: (...) -> List[Distribution] | |
""" | |
Return a list of installed Distribution objects. | |
If ``local_only`` is True (default), only return installations | |
local to the current virtualenv, if in a virtualenv. | |
``skip`` argument is an iterable of lower-case project names to | |
ignore; defaults to stdlib_pkgs | |
If ``include_editables`` is False, don't report editables. | |
If ``editables_only`` is True , only report editables. | |
If ``user_only`` is True , only report installations in the user | |
site directory. | |
If ``paths`` is set, only report the distributions present at the | |
specified list of locations. | |
""" | |
if paths: | |
working_set = pkg_resources.WorkingSet(paths) | |
else: | |
working_set = pkg_resources.working_set | |
if local_only: | |
local_test = dist_is_local | |
else: | |
def local_test(d): | |
return True | |
if include_editables: | |
def editable_test(d): | |
return True | |
else: | |
def editable_test(d): | |
return not dist_is_editable(d) | |
if editables_only: | |
def editables_only_test(d): | |
return dist_is_editable(d) | |
else: | |
def editables_only_test(d): | |
return True | |
if user_only: | |
user_test = dist_in_usersite | |
else: | |
def user_test(d): | |
return True | |
return [d for d in working_set | |
if local_test(d) and | |
d.key not in skip and | |
editable_test(d) and | |
editables_only_test(d) and | |
user_test(d) | |
] | |
def egg_link_path(dist): | |
# type: (Distribution) -> Optional[str] | |
""" | |
Return the path for the .egg-link file if it exists, otherwise, None. | |
There's 3 scenarios: | |
1) not in a virtualenv | |
try to find in site.USER_SITE, then site_packages | |
2) in a no-global virtualenv | |
try to find in site_packages | |
3) in a yes-global virtualenv | |
try to find in site_packages, then site.USER_SITE | |
(don't look in global location) | |
For #1 and #3, there could be odd cases, where there's an egg-link in 2 | |
locations. | |
This method will just return the first one found. | |
""" | |
sites = [] | |
if running_under_virtualenv(): | |
sites.append(site_packages) | |
if not virtualenv_no_global() and user_site: | |
sites.append(user_site) | |
else: | |
if user_site: | |
sites.append(user_site) | |
sites.append(site_packages) | |
for site in sites: | |
egglink = os.path.join(site, dist.project_name) + '.egg-link' | |
if os.path.isfile(egglink): | |
return egglink | |
return None | |
def dist_location(dist): | |
# type: (Distribution) -> str | |
""" | |
Get the site-packages location of this distribution. Generally | |
this is dist.location, except in the case of develop-installed | |
packages, where dist.location is the source code location, and we | |
want to know where the egg-link file is. | |
The returned location is normalized (in particular, with symlinks removed). | |
""" | |
egg_link = egg_link_path(dist) | |
if egg_link: | |
return normalize_path(egg_link) | |
return normalize_path(dist.location) | |
def write_output(msg, *args): | |
# type: (str, str) -> None | |
logger.info(msg, *args) | |
class FakeFile(object): | |
"""Wrap a list of lines in an object with readline() to make | |
ConfigParser happy.""" | |
def __init__(self, lines): | |
self._gen = (l for l in lines) | |
def readline(self): | |
try: | |
try: | |
return next(self._gen) | |
except NameError: | |
return self._gen.next() | |
except StopIteration: | |
return '' | |
def __iter__(self): | |
return self._gen | |
class StreamWrapper(StringIO): | |
@classmethod | |
def from_stream(cls, orig_stream): | |
cls.orig_stream = orig_stream | |
return cls() | |
# compileall.compile_dir() needs stdout.encoding to print to stdout | |
@property | |
def encoding(self): | |
return self.orig_stream.encoding | |
@contextlib.contextmanager | |
def captured_output(stream_name): | |
"""Return a context manager used by captured_stdout/stdin/stderr | |
that temporarily replaces the sys stream *stream_name* with a StringIO. | |
Taken from Lib/support/__init__.py in the CPython repo. | |
""" | |
orig_stdout = getattr(sys, stream_name) | |
setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout)) | |
try: | |
yield getattr(sys, stream_name) | |
finally: | |
setattr(sys, stream_name, orig_stdout) | |
def captured_stdout(): | |
"""Capture the output of sys.stdout: | |
with captured_stdout() as stdout: | |
print('hello') | |
self.assertEqual(stdout.getvalue(), 'hello\n') | |
Taken from Lib/support/__init__.py in the CPython repo. | |
""" | |
return captured_output('stdout') | |
def captured_stderr(): | |
""" | |
See captured_stdout(). | |
""" | |
return captured_output('stderr') | |
class cached_property(object): | |
"""A property that is only computed once per instance and then replaces | |
itself with an ordinary attribute. Deleting the attribute resets the | |
property. | |
Source: https://github.com/bottlepy/bottle/blob/0.11.5/bottle.py#L175 | |
""" | |
def __init__(self, func): | |
self.__doc__ = getattr(func, '__doc__') | |
self.func = func | |
def __get__(self, obj, cls): | |
if obj is None: | |
# We're being accessed from the class itself, not from an object | |
return self | |
value = obj.__dict__[self.func.__name__] = self.func(obj) | |
return value | |
def get_installed_version(dist_name, working_set=None): | |
"""Get the installed version of dist_name avoiding pkg_resources cache""" | |
# Create a requirement that we'll look for inside of setuptools. | |
req = pkg_resources.Requirement.parse(dist_name) | |
if working_set is None: | |
# We want to avoid having this cached, so we need to construct a new | |
# working set each time. | |
working_set = pkg_resources.WorkingSet() | |
# Get the installed distribution from our working set | |
dist = working_set.find(req) | |
# Check to see if we got an installed distribution or not, if we did | |
# we want to return it's version. | |
return dist.version if dist else None | |
def consume(iterator): | |
"""Consume an iterable at C speed.""" | |
deque(iterator, maxlen=0) | |
# Simulates an enum | |
def enum(*sequential, **named): | |
enums = dict(zip(sequential, range(len(sequential))), **named) | |
reverse = {value: key for key, value in enums.items()} | |
enums['reverse_mapping'] = reverse | |
return type('Enum', (), enums) | |
def build_netloc(host, port): | |
# type: (str, Optional[int]) -> str | |
""" | |
Build a netloc from a host-port pair | |
""" | |
if port is None: | |
return host | |
if ':' in host: | |
# Only wrap host with square brackets when it is IPv6 | |
host = '[{}]'.format(host) | |
return '{}:{}'.format(host, port) | |
def build_url_from_netloc(netloc, scheme='https'): | |
# type: (str, str) -> str | |
""" | |
Build a full URL from a netloc. | |
""" | |
if netloc.count(':') >= 2 and '@' not in netloc and '[' not in netloc: | |
# It must be a bare IPv6 address, so wrap it with brackets. | |
netloc = '[{}]'.format(netloc) | |
return '{}://{}'.format(scheme, netloc) | |
def parse_netloc(netloc): | |
# type: (str) -> Tuple[str, Optional[int]] | |
""" | |
Return the host-port pair from a netloc. | |
""" | |
url = build_url_from_netloc(netloc) | |
parsed = urllib_parse.urlparse(url) | |
return parsed.hostname, parsed.port | |
def split_auth_from_netloc(netloc): | |
""" | |
Parse out and remove the auth information from a netloc. | |
Returns: (netloc, (username, password)). | |
""" | |
if '@' not in netloc: | |
return netloc, (None, None) | |
# Split from the right because that's how urllib.parse.urlsplit() | |
# behaves if more than one @ is present (which can be checked using | |
# the password attribute of urlsplit()'s return value). | |
auth, netloc = netloc.rsplit('@', 1) | |
if ':' in auth: | |
# Split from the left because that's how urllib.parse.urlsplit() | |
# behaves if more than one : is present (which again can be checked | |
# using the password attribute of the return value) | |
user_pass = auth.split(':', 1) | |
else: | |
user_pass = auth, None | |
user_pass = tuple( | |
None if x is None else urllib_unquote(x) for x in user_pass | |
) | |
return netloc, user_pass | |
def redact_netloc(netloc): | |
# type: (str) -> str | |
""" | |
Replace the sensitive data in a netloc with "****", if it exists. | |
For example: | |
- "user:[email protected]" returns "user:****@example.com" | |
- "[email protected]" returns "****@example.com" | |
""" | |
netloc, (user, password) = split_auth_from_netloc(netloc) | |
if user is None: | |
return netloc | |
if password is None: | |
user = '****' | |
password = '' | |
else: | |
user = urllib_parse.quote(user) | |
password = ':****' | |
return '{user}{password}@{netloc}'.format(user=user, | |
password=password, | |
netloc=netloc) | |
def _transform_url(url, transform_netloc): | |
"""Transform and replace netloc in a url. | |
transform_netloc is a function taking the netloc and returning a | |
tuple. The first element of this tuple is the new netloc. The | |
entire tuple is returned. | |
Returns a tuple containing the transformed url as item 0 and the | |
original tuple returned by transform_netloc as item 1. | |
""" | |
purl = urllib_parse.urlsplit(url) | |
netloc_tuple = transform_netloc(purl.netloc) | |
# stripped url | |
url_pieces = ( | |
purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment | |
) | |
surl = urllib_parse.urlunsplit(url_pieces) | |
return surl, netloc_tuple | |
def _get_netloc(netloc): | |
return split_auth_from_netloc(netloc) | |
def _redact_netloc(netloc): | |
return (redact_netloc(netloc),) | |
def split_auth_netloc_from_url(url): | |
# type: (str) -> Tuple[str, str, Tuple[str, str]] | |
""" | |
Parse a url into separate netloc, auth, and url with no auth. | |
Returns: (url_without_auth, netloc, (username, password)) | |
""" | |
url_without_auth, (netloc, auth) = _transform_url(url, _get_netloc) | |
return url_without_auth, netloc, auth | |
def remove_auth_from_url(url): | |
# type: (str) -> str | |
"""Return a copy of url with 'username:password@' removed.""" | |
# username/pass params are passed to subversion through flags | |
# and are not recognized in the url. | |
return _transform_url(url, _get_netloc)[0] | |
def redact_auth_from_url(url): | |
# type: (str) -> str | |
"""Replace the password in a given url with ****.""" | |
return _transform_url(url, _redact_netloc)[0] | |
class HiddenText(object): | |
def __init__( | |
self, | |
secret, # type: str | |
redacted, # type: str | |
): | |
# type: (...) -> None | |
self.secret = secret | |
self.redacted = redacted | |
def __repr__(self): | |
# type: (...) -> str | |
return '<HiddenText {!r}>'.format(str(self)) | |
def __str__(self): | |
# type: (...) -> str | |
return self.redacted | |
# This is useful for testing. | |
def __eq__(self, other): | |
# type: (Any) -> bool | |
if type(self) != type(other): | |
return False | |
# The string being used for redaction doesn't also have to match, | |
# just the raw, original string. | |
return (self.secret == other.secret) | |
# We need to provide an explicit __ne__ implementation for Python 2. | |
# TODO: remove this when we drop PY2 support. | |
def __ne__(self, other): | |
# type: (Any) -> bool | |
return not self == other | |
def hide_value(value): | |
# type: (str) -> HiddenText | |
return HiddenText(value, redacted='****') | |
def hide_url(url): | |
# type: (str) -> HiddenText | |
redacted = redact_auth_from_url(url) | |
return HiddenText(url, redacted=redacted) | |
def protect_pip_from_modification_on_windows(modifying_pip): | |
# type: (bool) -> None | |
"""Protection of pip.exe from modification on Windows | |
On Windows, any operation modifying pip should be run as: | |
python -m pip ... | |
""" | |
pip_names = [ | |
"pip.exe", | |
"pip{}.exe".format(sys.version_info[0]), | |
"pip{}.{}.exe".format(*sys.version_info[:2]) | |
] | |
# See https://github.com/pypa/pip/issues/1299 for more discussion | |
should_show_use_python_msg = ( | |
modifying_pip and | |
WINDOWS and | |
os.path.basename(sys.argv[0]) in pip_names | |
) | |
if should_show_use_python_msg: | |
new_command = [ | |
sys.executable, "-m", "pip" | |
] + sys.argv[1:] | |
raise CommandError( | |
'To modify pip, please run the following command:\n{}' | |
.format(" ".join(new_command)) | |
) | |
def is_console_interactive(): | |
# type: () -> bool | |
"""Is this console interactive? | |
""" | |
return sys.stdin is not None and sys.stdin.isatty() | |
def hash_file(path, blocksize=1 << 20): | |
# type: (str, int) -> Tuple[Any, int] | |
"""Return (hash, length) for path using hashlib.sha256() | |
""" | |
h = hashlib.sha256() | |
length = 0 | |
with open(path, 'rb') as f: | |
for block in read_chunks(f, size=blocksize): | |
length += len(block) | |
h.update(block) | |
return h, length | |
def is_wheel_installed(): | |
""" | |
Return whether the wheel package is installed. | |
""" | |
try: | |
import wheel # noqa: F401 | |
except ImportError: | |
return False | |
return True |
"""Utilities for defining models | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
import operator | |
class KeyBasedCompareMixin(object): | |
"""Provides comparison capabilities that is based on a key | |
""" | |
def __init__(self, key, defining_class): | |
self._compare_key = key | |
self._defining_class = defining_class | |
def __hash__(self): | |
return hash(self._compare_key) | |
def __lt__(self, other): | |
return self._compare(other, operator.__lt__) | |
def __le__(self, other): | |
return self._compare(other, operator.__le__) | |
def __gt__(self, other): | |
return self._compare(other, operator.__gt__) | |
def __ge__(self, other): | |
return self._compare(other, operator.__ge__) | |
def __eq__(self, other): | |
return self._compare(other, operator.__eq__) | |
def __ne__(self, other): | |
return self._compare(other, operator.__ne__) | |
def _compare(self, other, method): | |
if not isinstance(other, self._defining_class): | |
return NotImplemented | |
return method(self._compare_key, other._compare_key) |
from __future__ import absolute_import | |
import logging | |
from email.parser import FeedParser | |
from pip._vendor import pkg_resources | |
from pip._vendor.packaging import specifiers, version | |
from pip._internal.exceptions import NoneMetadataError | |
from pip._internal.utils.misc import display_path | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Tuple | |
from email.message import Message | |
from pip._vendor.pkg_resources import Distribution | |
logger = logging.getLogger(__name__) | |
def check_requires_python(requires_python, version_info): | |
# type: (Optional[str], Tuple[int, ...]) -> bool | |
""" | |
Check if the given Python version matches a "Requires-Python" specifier. | |
:param version_info: A 3-tuple of ints representing a Python | |
major-minor-micro version to check (e.g. `sys.version_info[:3]`). | |
:return: `True` if the given Python version satisfies the requirement. | |
Otherwise, return `False`. | |
:raises InvalidSpecifier: If `requires_python` has an invalid format. | |
""" | |
if requires_python is None: | |
# The package provides no information | |
return True | |
requires_python_specifier = specifiers.SpecifierSet(requires_python) | |
python_version = version.parse('.'.join(map(str, version_info))) | |
return python_version in requires_python_specifier | |
def get_metadata(dist): | |
# type: (Distribution) -> Message | |
""" | |
:raises NoneMetadataError: if the distribution reports `has_metadata()` | |
True but `get_metadata()` returns None. | |
""" | |
metadata_name = 'METADATA' | |
if (isinstance(dist, pkg_resources.DistInfoDistribution) and | |
dist.has_metadata(metadata_name)): | |
metadata = dist.get_metadata(metadata_name) | |
elif dist.has_metadata('PKG-INFO'): | |
metadata_name = 'PKG-INFO' | |
metadata = dist.get_metadata(metadata_name) | |
else: | |
logger.warning("No metadata found in %s", display_path(dist.location)) | |
metadata = '' | |
if metadata is None: | |
raise NoneMetadataError(dist, metadata_name) | |
feed_parser = FeedParser() | |
# The following line errors out if with a "NoneType" TypeError if | |
# passed metadata=None. | |
feed_parser.feed(metadata) | |
return feed_parser.close() | |
def get_requires_python(dist): | |
# type: (pkg_resources.Distribution) -> Optional[str] | |
""" | |
Return the "Requires-Python" metadata for a distribution, or None | |
if not present. | |
""" | |
pkg_info_dict = get_metadata(dist) | |
requires_python = pkg_info_dict.get('Requires-Python') | |
if requires_python is not None: | |
# Convert to a str to satisfy the type checker, since requires_python | |
# can be a Header object. | |
requires_python = str(requires_python) | |
return requires_python | |
def get_installer(dist): | |
# type: (Distribution) -> str | |
if dist.has_metadata('INSTALLER'): | |
for line in dist.get_metadata_lines('INSTALLER'): | |
if line.strip(): | |
return line.strip() | |
return '' |
from pip._vendor.pkg_resources import yield_lines | |
from pip._vendor.six import ensure_str | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Dict, Iterable, List | |
class DictMetadata(object): | |
"""IMetadataProvider that reads metadata files from a dictionary. | |
""" | |
def __init__(self, metadata): | |
# type: (Dict[str, bytes]) -> None | |
self._metadata = metadata | |
def has_metadata(self, name): | |
# type: (str) -> bool | |
return name in self._metadata | |
def get_metadata(self, name): | |
# type: (str) -> str | |
try: | |
return ensure_str(self._metadata[name]) | |
except UnicodeDecodeError as e: | |
# Mirrors handling done in pkg_resources.NullProvider. | |
e.reason += " in {} file".format(name) | |
raise | |
def get_metadata_lines(self, name): | |
# type: (str) -> Iterable[str] | |
return yield_lines(self.get_metadata(name)) | |
def metadata_isdir(self, name): | |
# type: (str) -> bool | |
return False | |
def metadata_listdir(self, name): | |
# type: (str) -> List[str] | |
return [] | |
def run_script(self, script_name, namespace): | |
# type: (str, str) -> None | |
pass |
import sys | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional, Sequence | |
# Shim to wrap setup.py invocation with setuptools | |
# | |
# We set sys.argv[0] to the path to the underlying setup.py file so | |
# setuptools / distutils don't take the path to the setup.py to be "-c" when | |
# invoking via the shim. This avoids e.g. the following manifest_maker | |
# warning: "warning: manifest_maker: standard file '-c' not found". | |
_SETUPTOOLS_SHIM = ( | |
"import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};" | |
"f=getattr(tokenize, 'open', open)(__file__);" | |
"code=f.read().replace('\\r\\n', '\\n');" | |
"f.close();" | |
"exec(compile(code, __file__, 'exec'))" | |
) | |
def make_setuptools_shim_args( | |
setup_py_path, # type: str | |
global_options=None, # type: Sequence[str] | |
no_user_config=False, # type: bool | |
unbuffered_output=False # type: bool | |
): | |
# type: (...) -> List[str] | |
""" | |
Get setuptools command arguments with shim wrapped setup file invocation. | |
:param setup_py_path: The path to setup.py to be wrapped. | |
:param global_options: Additional global options. | |
:param no_user_config: If True, disables personal user configuration. | |
:param unbuffered_output: If True, adds the unbuffered switch to the | |
argument list. | |
""" | |
args = [sys.executable] | |
if unbuffered_output: | |
args += ["-u"] | |
args += ["-c", _SETUPTOOLS_SHIM.format(setup_py_path)] | |
if global_options: | |
args += global_options | |
if no_user_config: | |
args += ["--no-user-cfg"] | |
return args | |
def make_setuptools_bdist_wheel_args( | |
setup_py_path, # type: str | |
global_options, # type: Sequence[str] | |
build_options, # type: Sequence[str] | |
destination_dir, # type: str | |
): | |
# type: (...) -> List[str] | |
# NOTE: Eventually, we'd want to also -S to the flags here, when we're | |
# isolating. Currently, it breaks Python in virtualenvs, because it | |
# relies on site.py to find parts of the standard library outside the | |
# virtualenv. | |
args = make_setuptools_shim_args( | |
setup_py_path, | |
global_options=global_options, | |
unbuffered_output=True | |
) | |
args += ["bdist_wheel", "-d", destination_dir] | |
args += build_options | |
return args | |
def make_setuptools_clean_args( | |
setup_py_path, # type: str | |
global_options, # type: Sequence[str] | |
): | |
# type: (...) -> List[str] | |
args = make_setuptools_shim_args( | |
setup_py_path, | |
global_options=global_options, | |
unbuffered_output=True | |
) | |
args += ["clean", "--all"] | |
return args | |
def make_setuptools_develop_args( | |
setup_py_path, # type: str | |
global_options, # type: Sequence[str] | |
install_options, # type: Sequence[str] | |
no_user_config, # type: bool | |
prefix, # type: Optional[str] | |
home, # type: Optional[str] | |
use_user_site, # type: bool | |
): | |
# type: (...) -> List[str] | |
assert not (use_user_site and prefix) | |
args = make_setuptools_shim_args( | |
setup_py_path, | |
global_options=global_options, | |
no_user_config=no_user_config, | |
) | |
args += ["develop", "--no-deps"] | |
args += install_options | |
if prefix: | |
args += ["--prefix", prefix] | |
if home is not None: | |
args += ["--home", home] | |
if use_user_site: | |
args += ["--user", "--prefix="] | |
return args | |
def make_setuptools_egg_info_args( | |
setup_py_path, # type: str | |
egg_info_dir, # type: Optional[str] | |
no_user_config, # type: bool | |
): | |
# type: (...) -> List[str] | |
args = make_setuptools_shim_args(setup_py_path) | |
if no_user_config: | |
args += ["--no-user-cfg"] | |
args += ["egg_info"] | |
if egg_info_dir: | |
args += ["--egg-base", egg_info_dir] | |
return args | |
def make_setuptools_install_args( | |
setup_py_path, # type: str | |
global_options, # type: Sequence[str] | |
install_options, # type: Sequence[str] | |
record_filename, # type: str | |
root, # type: Optional[str] | |
prefix, # type: Optional[str] | |
header_dir, # type: Optional[str] | |
home, # type: Optional[str] | |
use_user_site, # type: bool | |
no_user_config, # type: bool | |
pycompile # type: bool | |
): | |
# type: (...) -> List[str] | |
assert not (use_user_site and prefix) | |
assert not (use_user_site and root) | |
args = make_setuptools_shim_args( | |
setup_py_path, | |
global_options=global_options, | |
no_user_config=no_user_config, | |
unbuffered_output=True | |
) | |
args += ["install", "--record", record_filename] | |
args += ["--single-version-externally-managed"] | |
if root is not None: | |
args += ["--root", root] | |
if prefix is not None: | |
args += ["--prefix", prefix] | |
if home is not None: | |
args += ["--home", home] | |
if use_user_site: | |
args += ["--user", "--prefix="] | |
if pycompile: | |
args += ["--compile"] | |
else: | |
args += ["--no-compile"] | |
if header_dir: | |
args += ["--install-headers", header_dir] | |
args += install_options | |
return args |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import subprocess | |
from pip._vendor.six.moves import shlex_quote | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.utils.compat import console_to_str, str_to_display | |
from pip._internal.utils.logging import subprocess_logger | |
from pip._internal.utils.misc import HiddenText, path_to_display | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.ui import open_spinner | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Callable, Iterable, List, Mapping, Optional, Text, Union, | |
) | |
from pip._internal.utils.ui import SpinnerInterface | |
CommandArgs = List[Union[str, HiddenText]] | |
LOG_DIVIDER = '----------------------------------------' | |
def make_command(*args): | |
# type: (Union[str, HiddenText, CommandArgs]) -> CommandArgs | |
""" | |
Create a CommandArgs object. | |
""" | |
command_args = [] # type: CommandArgs | |
for arg in args: | |
# Check for list instead of CommandArgs since CommandArgs is | |
# only known during type-checking. | |
if isinstance(arg, list): | |
command_args.extend(arg) | |
else: | |
# Otherwise, arg is str or HiddenText. | |
command_args.append(arg) | |
return command_args | |
def format_command_args(args): | |
# type: (Union[List[str], CommandArgs]) -> str | |
""" | |
Format command arguments for display. | |
""" | |
# For HiddenText arguments, display the redacted form by calling str(). | |
# Also, we don't apply str() to arguments that aren't HiddenText since | |
# this can trigger a UnicodeDecodeError in Python 2 if the argument | |
# has type unicode and includes a non-ascii character. (The type | |
# checker doesn't ensure the annotations are correct in all cases.) | |
return ' '.join( | |
shlex_quote(str(arg)) if isinstance(arg, HiddenText) | |
else shlex_quote(arg) for arg in args | |
) | |
def reveal_command_args(args): | |
# type: (Union[List[str], CommandArgs]) -> List[str] | |
""" | |
Return the arguments in their raw, unredacted form. | |
""" | |
return [ | |
arg.secret if isinstance(arg, HiddenText) else arg for arg in args | |
] | |
def make_subprocess_output_error( | |
cmd_args, # type: Union[List[str], CommandArgs] | |
cwd, # type: Optional[str] | |
lines, # type: List[Text] | |
exit_status, # type: int | |
): | |
# type: (...) -> Text | |
""" | |
Create and return the error message to use to log a subprocess error | |
with command output. | |
:param lines: A list of lines, each ending with a newline. | |
""" | |
command = format_command_args(cmd_args) | |
# Convert `command` and `cwd` to text (unicode in Python 2) so we can use | |
# them as arguments in the unicode format string below. This avoids | |
# "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2 | |
# if either contains a non-ascii character. | |
command_display = str_to_display(command, desc='command bytes') | |
cwd_display = path_to_display(cwd) | |
# We know the joined output value ends in a newline. | |
output = ''.join(lines) | |
msg = ( | |
# Use a unicode string to avoid "UnicodeEncodeError: 'ascii' | |
# codec can't encode character ..." in Python 2 when a format | |
# argument (e.g. `output`) has a non-ascii character. | |
u'Command errored out with exit status {exit_status}:\n' | |
' command: {command_display}\n' | |
' cwd: {cwd_display}\n' | |
'Complete output ({line_count} lines):\n{output}{divider}' | |
).format( | |
exit_status=exit_status, | |
command_display=command_display, | |
cwd_display=cwd_display, | |
line_count=len(lines), | |
output=output, | |
divider=LOG_DIVIDER, | |
) | |
return msg | |
def call_subprocess( | |
cmd, # type: Union[List[str], CommandArgs] | |
show_stdout=False, # type: bool | |
cwd=None, # type: Optional[str] | |
on_returncode='raise', # type: str | |
extra_ok_returncodes=None, # type: Optional[Iterable[int]] | |
command_desc=None, # type: Optional[str] | |
extra_environ=None, # type: Optional[Mapping[str, Any]] | |
unset_environ=None, # type: Optional[Iterable[str]] | |
spinner=None, # type: Optional[SpinnerInterface] | |
log_failed_cmd=True # type: Optional[bool] | |
): | |
# type: (...) -> Text | |
""" | |
Args: | |
show_stdout: if true, use INFO to log the subprocess's stderr and | |
stdout streams. Otherwise, use DEBUG. Defaults to False. | |
extra_ok_returncodes: an iterable of integer return codes that are | |
acceptable, in addition to 0. Defaults to None, which means []. | |
unset_environ: an iterable of environment variable names to unset | |
prior to calling subprocess.Popen(). | |
log_failed_cmd: if false, failed commands are not logged, only raised. | |
""" | |
if extra_ok_returncodes is None: | |
extra_ok_returncodes = [] | |
if unset_environ is None: | |
unset_environ = [] | |
# Most places in pip use show_stdout=False. What this means is-- | |
# | |
# - We connect the child's output (combined stderr and stdout) to a | |
# single pipe, which we read. | |
# - We log this output to stderr at DEBUG level as it is received. | |
# - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't | |
# requested), then we show a spinner so the user can still see the | |
# subprocess is in progress. | |
# - If the subprocess exits with an error, we log the output to stderr | |
# at ERROR level if it hasn't already been displayed to the console | |
# (e.g. if --verbose logging wasn't enabled). This way we don't log | |
# the output to the console twice. | |
# | |
# If show_stdout=True, then the above is still done, but with DEBUG | |
# replaced by INFO. | |
if show_stdout: | |
# Then log the subprocess output at INFO level. | |
log_subprocess = subprocess_logger.info | |
used_level = logging.INFO | |
else: | |
# Then log the subprocess output using DEBUG. This also ensures | |
# it will be logged to the log file (aka user_log), if enabled. | |
log_subprocess = subprocess_logger.debug | |
used_level = logging.DEBUG | |
# Whether the subprocess will be visible in the console. | |
showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level | |
# Only use the spinner if we're not showing the subprocess output | |
# and we have a spinner. | |
use_spinner = not showing_subprocess and spinner is not None | |
if command_desc is None: | |
command_desc = format_command_args(cmd) | |
log_subprocess("Running command %s", command_desc) | |
env = os.environ.copy() | |
if extra_environ: | |
env.update(extra_environ) | |
for name in unset_environ: | |
env.pop(name, None) | |
try: | |
proc = subprocess.Popen( | |
# Convert HiddenText objects to the underlying str. | |
reveal_command_args(cmd), | |
stderr=subprocess.STDOUT, stdin=subprocess.PIPE, | |
stdout=subprocess.PIPE, cwd=cwd, env=env, | |
) | |
proc.stdin.close() | |
except Exception as exc: | |
if log_failed_cmd: | |
subprocess_logger.critical( | |
"Error %s while executing command %s", exc, command_desc, | |
) | |
raise | |
all_output = [] | |
while True: | |
# The "line" value is a unicode string in Python 2. | |
line = console_to_str(proc.stdout.readline()) | |
if not line: | |
break | |
line = line.rstrip() | |
all_output.append(line + '\n') | |
# Show the line immediately. | |
log_subprocess(line) | |
# Update the spinner. | |
if use_spinner: | |
spinner.spin() | |
try: | |
proc.wait() | |
finally: | |
if proc.stdout: | |
proc.stdout.close() | |
proc_had_error = ( | |
proc.returncode and proc.returncode not in extra_ok_returncodes | |
) | |
if use_spinner: | |
if proc_had_error: | |
spinner.finish("error") | |
else: | |
spinner.finish("done") | |
if proc_had_error: | |
if on_returncode == 'raise': | |
if not showing_subprocess and log_failed_cmd: | |
# Then the subprocess streams haven't been logged to the | |
# console yet. | |
msg = make_subprocess_output_error( | |
cmd_args=cmd, | |
cwd=cwd, | |
lines=all_output, | |
exit_status=proc.returncode, | |
) | |
subprocess_logger.error(msg) | |
exc_msg = ( | |
'Command errored out with exit status {}: {} ' | |
'Check the logs for full command output.' | |
).format(proc.returncode, command_desc) | |
raise InstallationError(exc_msg) | |
elif on_returncode == 'warn': | |
subprocess_logger.warning( | |
'Command "%s" had error code %s in %s', | |
command_desc, proc.returncode, cwd, | |
) | |
elif on_returncode == 'ignore': | |
pass | |
else: | |
raise ValueError('Invalid value: on_returncode=%s' % | |
repr(on_returncode)) | |
return ''.join(all_output) | |
def runner_with_spinner_message(message): | |
# type: (str) -> Callable[..., None] | |
"""Provide a subprocess_runner that shows a spinner message. | |
Intended for use with for pep517's Pep517HookCaller. Thus, the runner has | |
an API that matches what's expected by Pep517HookCaller.subprocess_runner. | |
""" | |
def runner( | |
cmd, # type: List[str] | |
cwd=None, # type: Optional[str] | |
extra_environ=None # type: Optional[Mapping[str, Any]] | |
): | |
# type: (...) -> None | |
with open_spinner(message) as spinner: | |
call_subprocess( | |
cmd, | |
cwd=cwd, | |
extra_environ=extra_environ, | |
spinner=spinner, | |
) | |
return runner |
from __future__ import absolute_import | |
import errno | |
import itertools | |
import logging | |
import os.path | |
import tempfile | |
from contextlib import contextmanager | |
from pip._vendor.contextlib2 import ExitStack | |
from pip._internal.utils.misc import rmtree | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Dict, Iterator, Optional, TypeVar | |
_T = TypeVar('_T', bound='TempDirectory') | |
logger = logging.getLogger(__name__) | |
_tempdir_manager = None # type: Optional[ExitStack] | |
@contextmanager | |
def global_tempdir_manager(): | |
# type: () -> Iterator[None] | |
global _tempdir_manager | |
with ExitStack() as stack: | |
old_tempdir_manager, _tempdir_manager = _tempdir_manager, stack | |
try: | |
yield | |
finally: | |
_tempdir_manager = old_tempdir_manager | |
class TempDirectoryTypeRegistry(object): | |
"""Manages temp directory behavior | |
""" | |
def __init__(self): | |
# type: () -> None | |
self._should_delete = {} # type: Dict[str, bool] | |
def set_delete(self, kind, value): | |
# type: (str, bool) -> None | |
"""Indicate whether a TempDirectory of the given kind should be | |
auto-deleted. | |
""" | |
self._should_delete[kind] = value | |
def get_delete(self, kind): | |
# type: (str) -> bool | |
"""Get configured auto-delete flag for a given TempDirectory type, | |
default True. | |
""" | |
return self._should_delete.get(kind, True) | |
_tempdir_registry = None # type: Optional[TempDirectoryTypeRegistry] | |
@contextmanager | |
def tempdir_registry(): | |
# type: () -> Iterator[TempDirectoryTypeRegistry] | |
"""Provides a scoped global tempdir registry that can be used to dictate | |
whether directories should be deleted. | |
""" | |
global _tempdir_registry | |
old_tempdir_registry = _tempdir_registry | |
_tempdir_registry = TempDirectoryTypeRegistry() | |
try: | |
yield _tempdir_registry | |
finally: | |
_tempdir_registry = old_tempdir_registry | |
class TempDirectory(object): | |
"""Helper class that owns and cleans up a temporary directory. | |
This class can be used as a context manager or as an OO representation of a | |
temporary directory. | |
Attributes: | |
path | |
Location to the created temporary directory | |
delete | |
Whether the directory should be deleted when exiting | |
(when used as a contextmanager) | |
Methods: | |
cleanup() | |
Deletes the temporary directory | |
When used as a context manager, if the delete attribute is True, on | |
exiting the context the temporary directory is deleted. | |
""" | |
def __init__( | |
self, | |
path=None, # type: Optional[str] | |
delete=None, # type: Optional[bool] | |
kind="temp", # type: str | |
globally_managed=False, # type: bool | |
): | |
super(TempDirectory, self).__init__() | |
# If we were given an explicit directory, resolve delete option now. | |
# Otherwise we wait until cleanup and see what tempdir_registry says. | |
if path is not None and delete is None: | |
delete = False | |
if path is None: | |
path = self._create(kind) | |
self._path = path | |
self._deleted = False | |
self.delete = delete | |
self.kind = kind | |
if globally_managed: | |
assert _tempdir_manager is not None | |
_tempdir_manager.enter_context(self) | |
@property | |
def path(self): | |
# type: () -> str | |
assert not self._deleted, ( | |
"Attempted to access deleted path: {}".format(self._path) | |
) | |
return self._path | |
def __repr__(self): | |
# type: () -> str | |
return "<{} {!r}>".format(self.__class__.__name__, self.path) | |
def __enter__(self): | |
# type: (_T) -> _T | |
return self | |
def __exit__(self, exc, value, tb): | |
# type: (Any, Any, Any) -> None | |
if self.delete is not None: | |
delete = self.delete | |
elif _tempdir_registry: | |
delete = _tempdir_registry.get_delete(self.kind) | |
else: | |
delete = True | |
if delete: | |
self.cleanup() | |
def _create(self, kind): | |
# type: (str) -> str | |
"""Create a temporary directory and store its path in self.path | |
""" | |
# We realpath here because some systems have their default tmpdir | |
# symlinked to another directory. This tends to confuse build | |
# scripts, so we canonicalize the path by traversing potential | |
# symlinks here. | |
path = os.path.realpath( | |
tempfile.mkdtemp(prefix="pip-{}-".format(kind)) | |
) | |
logger.debug("Created temporary directory: {}".format(path)) | |
return path | |
def cleanup(self): | |
# type: () -> None | |
"""Remove the temporary directory created and reset state | |
""" | |
self._deleted = True | |
if os.path.exists(self._path): | |
rmtree(self._path) | |
class AdjacentTempDirectory(TempDirectory): | |
"""Helper class that creates a temporary directory adjacent to a real one. | |
Attributes: | |
original | |
The original directory to create a temp directory for. | |
path | |
After calling create() or entering, contains the full | |
path to the temporary directory. | |
delete | |
Whether the directory should be deleted when exiting | |
(when used as a contextmanager) | |
""" | |
# The characters that may be used to name the temp directory | |
# We always prepend a ~ and then rotate through these until | |
# a usable name is found. | |
# pkg_resources raises a different error for .dist-info folder | |
# with leading '-' and invalid metadata | |
LEADING_CHARS = "-~.=%0123456789" | |
def __init__(self, original, delete=None): | |
# type: (str, Optional[bool]) -> None | |
self.original = original.rstrip('/\\') | |
super(AdjacentTempDirectory, self).__init__(delete=delete) | |
@classmethod | |
def _generate_names(cls, name): | |
# type: (str) -> Iterator[str] | |
"""Generates a series of temporary names. | |
The algorithm replaces the leading characters in the name | |
with ones that are valid filesystem characters, but are not | |
valid package names (for both Python and pip definitions of | |
package). | |
""" | |
for i in range(1, len(name)): | |
for candidate in itertools.combinations_with_replacement( | |
cls.LEADING_CHARS, i - 1): | |
new_name = '~' + ''.join(candidate) + name[i:] | |
if new_name != name: | |
yield new_name | |
# If we make it this far, we will have to make a longer name | |
for i in range(len(cls.LEADING_CHARS)): | |
for candidate in itertools.combinations_with_replacement( | |
cls.LEADING_CHARS, i): | |
new_name = '~' + ''.join(candidate) + name | |
if new_name != name: | |
yield new_name | |
def _create(self, kind): | |
# type: (str) -> str | |
root, name = os.path.split(self.original) | |
for candidate in self._generate_names(name): | |
path = os.path.join(root, candidate) | |
try: | |
os.mkdir(path) | |
except OSError as ex: | |
# Continue if the name exists already | |
if ex.errno != errno.EEXIST: | |
raise | |
else: | |
path = os.path.realpath(path) | |
break | |
else: | |
# Final fallback on the default behavior. | |
path = os.path.realpath( | |
tempfile.mkdtemp(prefix="pip-{}-".format(kind)) | |
) | |
logger.debug("Created temporary directory: {}".format(path)) | |
return path |
"""For neatly implementing static typing in pip. | |
`mypy` - the static type analysis tool we use - uses the `typing` module, which | |
provides core functionality fundamental to mypy's functioning. | |
Generally, `typing` would be imported at runtime and used in that fashion - | |
it acts as a no-op at runtime and does not have any run-time overhead by | |
design. | |
As it turns out, `typing` is not vendorable - it uses separate sources for | |
Python 2/Python 3. Thus, this codebase can not expect it to be present. | |
To work around this, mypy allows the typing import to be behind a False-y | |
optional to prevent it from running at runtime and type-comments can be used | |
to remove the need for the types to be accessible directly during runtime. | |
This module provides the False-y guard in a nicely named fashion so that a | |
curious maintainer can reach here to read this. | |
In pip, all static-typing related imports should be guarded as follows: | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import ... | |
Ref: https://github.com/python/mypy/issues/3216 | |
""" | |
MYPY_CHECK_RUNNING = False | |
if MYPY_CHECK_RUNNING: | |
from typing import cast | |
else: | |
# typing's cast() is needed at runtime, but we don't want to import typing. | |
# Thus, we use a dummy no-op version, which we tell mypy to ignore. | |
def cast(type_, value): # type: ignore | |
return value |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import, division | |
import contextlib | |
import itertools | |
import logging | |
import sys | |
import time | |
from signal import SIGINT, default_int_handler, signal | |
from pip._vendor import six | |
from pip._vendor.progress import HIDE_CURSOR, SHOW_CURSOR | |
from pip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar | |
from pip._vendor.progress.spinner import Spinner | |
from pip._internal.utils.compat import WINDOWS | |
from pip._internal.utils.logging import get_indentation | |
from pip._internal.utils.misc import format_size | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Any, Iterator, IO | |
try: | |
from pip._vendor import colorama | |
# Lots of different errors can come from this, including SystemError and | |
# ImportError. | |
except Exception: | |
colorama = None | |
logger = logging.getLogger(__name__) | |
def _select_progress_class(preferred, fallback): | |
encoding = getattr(preferred.file, "encoding", None) | |
# If we don't know what encoding this file is in, then we'll just assume | |
# that it doesn't support unicode and use the ASCII bar. | |
if not encoding: | |
return fallback | |
# Collect all of the possible characters we want to use with the preferred | |
# bar. | |
characters = [ | |
getattr(preferred, "empty_fill", six.text_type()), | |
getattr(preferred, "fill", six.text_type()), | |
] | |
characters += list(getattr(preferred, "phases", [])) | |
# Try to decode the characters we're using for the bar using the encoding | |
# of the given file, if this works then we'll assume that we can use the | |
# fancier bar and if not we'll fall back to the plaintext bar. | |
try: | |
six.text_type().join(characters).encode(encoding) | |
except UnicodeEncodeError: | |
return fallback | |
else: | |
return preferred | |
_BaseBar = _select_progress_class(IncrementalBar, Bar) # type: Any | |
class InterruptibleMixin(object): | |
""" | |
Helper to ensure that self.finish() gets called on keyboard interrupt. | |
This allows downloads to be interrupted without leaving temporary state | |
(like hidden cursors) behind. | |
This class is similar to the progress library's existing SigIntMixin | |
helper, but as of version 1.2, that helper has the following problems: | |
1. It calls sys.exit(). | |
2. It discards the existing SIGINT handler completely. | |
3. It leaves its own handler in place even after an uninterrupted finish, | |
which will have unexpected delayed effects if the user triggers an | |
unrelated keyboard interrupt some time after a progress-displaying | |
download has already completed, for example. | |
""" | |
def __init__(self, *args, **kwargs): | |
""" | |
Save the original SIGINT handler for later. | |
""" | |
super(InterruptibleMixin, self).__init__(*args, **kwargs) | |
self.original_handler = signal(SIGINT, self.handle_sigint) | |
# If signal() returns None, the previous handler was not installed from | |
# Python, and we cannot restore it. This probably should not happen, | |
# but if it does, we must restore something sensible instead, at least. | |
# The least bad option should be Python's default SIGINT handler, which | |
# just raises KeyboardInterrupt. | |
if self.original_handler is None: | |
self.original_handler = default_int_handler | |
def finish(self): | |
""" | |
Restore the original SIGINT handler after finishing. | |
This should happen regardless of whether the progress display finishes | |
normally, or gets interrupted. | |
""" | |
super(InterruptibleMixin, self).finish() | |
signal(SIGINT, self.original_handler) | |
def handle_sigint(self, signum, frame): | |
""" | |
Call self.finish() before delegating to the original SIGINT handler. | |
This handler should only be in place while the progress display is | |
active. | |
""" | |
self.finish() | |
self.original_handler(signum, frame) | |
class SilentBar(Bar): | |
def update(self): | |
pass | |
class BlueEmojiBar(IncrementalBar): | |
suffix = "%(percent)d%%" | |
bar_prefix = " " | |
bar_suffix = " " | |
phases = (u"\U0001F539", u"\U0001F537", u"\U0001F535") # type: Any | |
class DownloadProgressMixin(object): | |
def __init__(self, *args, **kwargs): | |
super(DownloadProgressMixin, self).__init__(*args, **kwargs) | |
self.message = (" " * (get_indentation() + 2)) + self.message | |
@property | |
def downloaded(self): | |
return format_size(self.index) | |
@property | |
def download_speed(self): | |
# Avoid zero division errors... | |
if self.avg == 0.0: | |
return "..." | |
return format_size(1 / self.avg) + "/s" | |
@property | |
def pretty_eta(self): | |
if self.eta: | |
return "eta %s" % self.eta_td | |
return "" | |
def iter(self, it): | |
for x in it: | |
yield x | |
self.next(len(x)) | |
self.finish() | |
class WindowsMixin(object): | |
def __init__(self, *args, **kwargs): | |
# The Windows terminal does not support the hide/show cursor ANSI codes | |
# even with colorama. So we'll ensure that hide_cursor is False on | |
# Windows. | |
# This call needs to go before the super() call, so that hide_cursor | |
# is set in time. The base progress bar class writes the "hide cursor" | |
# code to the terminal in its init, so if we don't set this soon | |
# enough, we get a "hide" with no corresponding "show"... | |
if WINDOWS and self.hide_cursor: | |
self.hide_cursor = False | |
super(WindowsMixin, self).__init__(*args, **kwargs) | |
# Check if we are running on Windows and we have the colorama module, | |
# if we do then wrap our file with it. | |
if WINDOWS and colorama: | |
self.file = colorama.AnsiToWin32(self.file) | |
# The progress code expects to be able to call self.file.isatty() | |
# but the colorama.AnsiToWin32() object doesn't have that, so we'll | |
# add it. | |
self.file.isatty = lambda: self.file.wrapped.isatty() | |
# The progress code expects to be able to call self.file.flush() | |
# but the colorama.AnsiToWin32() object doesn't have that, so we'll | |
# add it. | |
self.file.flush = lambda: self.file.wrapped.flush() | |
class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, | |
DownloadProgressMixin): | |
file = sys.stdout | |
message = "%(percent)d%%" | |
suffix = "%(downloaded)s %(download_speed)s %(pretty_eta)s" | |
# NOTE: The "type: ignore" comments on the following classes are there to | |
# work around https://github.com/python/typing/issues/241 | |
class DefaultDownloadProgressBar(BaseDownloadProgressBar, | |
_BaseBar): | |
pass | |
class DownloadSilentBar(BaseDownloadProgressBar, SilentBar): # type: ignore | |
pass | |
class DownloadBar(BaseDownloadProgressBar, # type: ignore | |
Bar): | |
pass | |
class DownloadFillingCirclesBar(BaseDownloadProgressBar, # type: ignore | |
FillingCirclesBar): | |
pass | |
class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar, # type: ignore | |
BlueEmojiBar): | |
pass | |
class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin, | |
DownloadProgressMixin, Spinner): | |
file = sys.stdout | |
suffix = "%(downloaded)s %(download_speed)s" | |
def next_phase(self): | |
if not hasattr(self, "_phaser"): | |
self._phaser = itertools.cycle(self.phases) | |
return next(self._phaser) | |
def update(self): | |
message = self.message % self | |
phase = self.next_phase() | |
suffix = self.suffix % self | |
line = ''.join([ | |
message, | |
" " if message else "", | |
phase, | |
" " if suffix else "", | |
suffix, | |
]) | |
self.writeln(line) | |
BAR_TYPES = { | |
"off": (DownloadSilentBar, DownloadSilentBar), | |
"on": (DefaultDownloadProgressBar, DownloadProgressSpinner), | |
"ascii": (DownloadBar, DownloadProgressSpinner), | |
"pretty": (DownloadFillingCirclesBar, DownloadProgressSpinner), | |
"emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner) | |
} | |
def DownloadProgressProvider(progress_bar, max=None): | |
if max is None or max == 0: | |
return BAR_TYPES[progress_bar][1]().iter | |
else: | |
return BAR_TYPES[progress_bar][0](max=max).iter | |
################################################################ | |
# Generic "something is happening" spinners | |
# | |
# We don't even try using progress.spinner.Spinner here because it's actually | |
# simpler to reimplement from scratch than to coerce their code into doing | |
# what we need. | |
################################################################ | |
@contextlib.contextmanager | |
def hidden_cursor(file): | |
# type: (IO[Any]) -> Iterator[None] | |
# The Windows terminal does not support the hide/show cursor ANSI codes, | |
# even via colorama. So don't even try. | |
if WINDOWS: | |
yield | |
# We don't want to clutter the output with control characters if we're | |
# writing to a file, or if the user is running with --quiet. | |
# See https://github.com/pypa/pip/issues/3418 | |
elif not file.isatty() or logger.getEffectiveLevel() > logging.INFO: | |
yield | |
else: | |
file.write(HIDE_CURSOR) | |
try: | |
yield | |
finally: | |
file.write(SHOW_CURSOR) | |
class RateLimiter(object): | |
def __init__(self, min_update_interval_seconds): | |
# type: (float) -> None | |
self._min_update_interval_seconds = min_update_interval_seconds | |
self._last_update = 0 # type: float | |
def ready(self): | |
# type: () -> bool | |
now = time.time() | |
delta = now - self._last_update | |
return delta >= self._min_update_interval_seconds | |
def reset(self): | |
# type: () -> None | |
self._last_update = time.time() | |
class SpinnerInterface(object): | |
def spin(self): | |
# type: () -> None | |
raise NotImplementedError() | |
def finish(self, final_status): | |
# type: (str) -> None | |
raise NotImplementedError() | |
class InteractiveSpinner(SpinnerInterface): | |
def __init__(self, message, file=None, spin_chars="-\\|/", | |
# Empirically, 8 updates/second looks nice | |
min_update_interval_seconds=0.125): | |
self._message = message | |
if file is None: | |
file = sys.stdout | |
self._file = file | |
self._rate_limiter = RateLimiter(min_update_interval_seconds) | |
self._finished = False | |
self._spin_cycle = itertools.cycle(spin_chars) | |
self._file.write(" " * get_indentation() + self._message + " ... ") | |
self._width = 0 | |
def _write(self, status): | |
assert not self._finished | |
# Erase what we wrote before by backspacing to the beginning, writing | |
# spaces to overwrite the old text, and then backspacing again | |
backup = "\b" * self._width | |
self._file.write(backup + " " * self._width + backup) | |
# Now we have a blank slate to add our status | |
self._file.write(status) | |
self._width = len(status) | |
self._file.flush() | |
self._rate_limiter.reset() | |
def spin(self): | |
# type: () -> None | |
if self._finished: | |
return | |
if not self._rate_limiter.ready(): | |
return | |
self._write(next(self._spin_cycle)) | |
def finish(self, final_status): | |
# type: (str) -> None | |
if self._finished: | |
return | |
self._write(final_status) | |
self._file.write("\n") | |
self._file.flush() | |
self._finished = True | |
# Used for dumb terminals, non-interactive installs (no tty), etc. | |
# We still print updates occasionally (once every 60 seconds by default) to | |
# act as a keep-alive for systems like Travis-CI that take lack-of-output as | |
# an indication that a task has frozen. | |
class NonInteractiveSpinner(SpinnerInterface): | |
def __init__(self, message, min_update_interval_seconds=60): | |
# type: (str, float) -> None | |
self._message = message | |
self._finished = False | |
self._rate_limiter = RateLimiter(min_update_interval_seconds) | |
self._update("started") | |
def _update(self, status): | |
assert not self._finished | |
self._rate_limiter.reset() | |
logger.info("%s: %s", self._message, status) | |
def spin(self): | |
# type: () -> None | |
if self._finished: | |
return | |
if not self._rate_limiter.ready(): | |
return | |
self._update("still running...") | |
def finish(self, final_status): | |
# type: (str) -> None | |
if self._finished: | |
return | |
self._update("finished with status '%s'" % (final_status,)) | |
self._finished = True | |
@contextlib.contextmanager | |
def open_spinner(message): | |
# type: (str) -> Iterator[SpinnerInterface] | |
# Interactive spinner goes directly to sys.stdout rather than being routed | |
# through the logging system, but it acts like it has level INFO, | |
# i.e. it's only displayed if we're at level INFO or better. | |
# Non-interactive spinner goes through the logging system, so it is always | |
# in sync with logging configuration. | |
if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: | |
spinner = InteractiveSpinner(message) # type: SpinnerInterface | |
else: | |
spinner = NonInteractiveSpinner(message) | |
try: | |
with hidden_cursor(sys.stdout): | |
yield spinner | |
except KeyboardInterrupt: | |
spinner.finish("canceled") | |
raise | |
except Exception: | |
spinner.finish("error") | |
raise | |
else: | |
spinner.finish("done") |
"""Utilities related archives. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import shutil | |
import stat | |
import tarfile | |
import zipfile | |
from pip._internal.exceptions import InstallationError | |
from pip._internal.utils.filetypes import ( | |
BZ2_EXTENSIONS, | |
TAR_EXTENSIONS, | |
XZ_EXTENSIONS, | |
ZIP_EXTENSIONS, | |
) | |
from pip._internal.utils.misc import ensure_dir | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Iterable, List, Optional, Text, Union | |
logger = logging.getLogger(__name__) | |
SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS | |
try: | |
import bz2 # noqa | |
SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS | |
except ImportError: | |
logger.debug('bz2 module is not available') | |
try: | |
# Only for Python 3.3+ | |
import lzma # noqa | |
SUPPORTED_EXTENSIONS += XZ_EXTENSIONS | |
except ImportError: | |
logger.debug('lzma module is not available') | |
def current_umask(): | |
"""Get the current umask which involves having to set it temporarily.""" | |
mask = os.umask(0) | |
os.umask(mask) | |
return mask | |
def split_leading_dir(path): | |
# type: (Union[str, Text]) -> List[Union[str, Text]] | |
path = path.lstrip('/').lstrip('\\') | |
if ( | |
'/' in path and ( | |
('\\' in path and path.find('/') < path.find('\\')) or | |
'\\' not in path | |
) | |
): | |
return path.split('/', 1) | |
elif '\\' in path: | |
return path.split('\\', 1) | |
else: | |
return [path, ''] | |
def has_leading_dir(paths): | |
# type: (Iterable[Union[str, Text]]) -> bool | |
"""Returns true if all the paths have the same leading path name | |
(i.e., everything is in one subdirectory in an archive)""" | |
common_prefix = None | |
for path in paths: | |
prefix, rest = split_leading_dir(path) | |
if not prefix: | |
return False | |
elif common_prefix is None: | |
common_prefix = prefix | |
elif prefix != common_prefix: | |
return False | |
return True | |
def is_within_directory(directory, target): | |
# type: ((Union[str, Text]), (Union[str, Text])) -> bool | |
""" | |
Return true if the absolute path of target is within the directory | |
""" | |
abs_directory = os.path.abspath(directory) | |
abs_target = os.path.abspath(target) | |
prefix = os.path.commonprefix([abs_directory, abs_target]) | |
return prefix == abs_directory | |
def unzip_file(filename, location, flatten=True): | |
# type: (str, str, bool) -> None | |
""" | |
Unzip the file (with path `filename`) to the destination `location`. All | |
files are written based on system defaults and umask (i.e. permissions are | |
not preserved), except that regular file members with any execute | |
permissions (user, group, or world) have "chmod +x" applied after being | |
written. Note that for windows, any execute changes using os.chmod are | |
no-ops per the python docs. | |
""" | |
ensure_dir(location) | |
zipfp = open(filename, 'rb') | |
try: | |
zip = zipfile.ZipFile(zipfp, allowZip64=True) | |
leading = has_leading_dir(zip.namelist()) and flatten | |
for info in zip.infolist(): | |
name = info.filename | |
fn = name | |
if leading: | |
fn = split_leading_dir(name)[1] | |
fn = os.path.join(location, fn) | |
dir = os.path.dirname(fn) | |
if not is_within_directory(location, fn): | |
message = ( | |
'The zip file ({}) has a file ({}) trying to install ' | |
'outside target directory ({})' | |
) | |
raise InstallationError(message.format(filename, fn, location)) | |
if fn.endswith('/') or fn.endswith('\\'): | |
# A directory | |
ensure_dir(fn) | |
else: | |
ensure_dir(dir) | |
# Don't use read() to avoid allocating an arbitrarily large | |
# chunk of memory for the file's content | |
fp = zip.open(name) | |
try: | |
with open(fn, 'wb') as destfp: | |
shutil.copyfileobj(fp, destfp) | |
finally: | |
fp.close() | |
mode = info.external_attr >> 16 | |
# if mode and regular file and any execute permissions for | |
# user/group/world? | |
if mode and stat.S_ISREG(mode) and mode & 0o111: | |
# make dest file have execute for user/group/world | |
# (chmod +x) no-op on windows per python docs | |
os.chmod(fn, (0o777 - current_umask() | 0o111)) | |
finally: | |
zipfp.close() | |
def untar_file(filename, location): | |
# type: (str, str) -> None | |
""" | |
Untar the file (with path `filename`) to the destination `location`. | |
All files are written based on system defaults and umask (i.e. permissions | |
are not preserved), except that regular file members with any execute | |
permissions (user, group, or world) have "chmod +x" applied after being | |
written. Note that for windows, any execute changes using os.chmod are | |
no-ops per the python docs. | |
""" | |
ensure_dir(location) | |
if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): | |
mode = 'r:gz' | |
elif filename.lower().endswith(BZ2_EXTENSIONS): | |
mode = 'r:bz2' | |
elif filename.lower().endswith(XZ_EXTENSIONS): | |
mode = 'r:xz' | |
elif filename.lower().endswith('.tar'): | |
mode = 'r' | |
else: | |
logger.warning( | |
'Cannot determine compression type for file %s', filename, | |
) | |
mode = 'r:*' | |
tar = tarfile.open(filename, mode) | |
try: | |
leading = has_leading_dir([ | |
member.name for member in tar.getmembers() | |
]) | |
for member in tar.getmembers(): | |
fn = member.name | |
if leading: | |
# https://github.com/python/mypy/issues/1174 | |
fn = split_leading_dir(fn)[1] # type: ignore | |
path = os.path.join(location, fn) | |
if not is_within_directory(location, path): | |
message = ( | |
'The tar file ({}) has a file ({}) trying to install ' | |
'outside target directory ({})' | |
) | |
raise InstallationError( | |
message.format(filename, path, location) | |
) | |
if member.isdir(): | |
ensure_dir(path) | |
elif member.issym(): | |
try: | |
# https://github.com/python/typeshed/issues/2673 | |
tar._extract_member(member, path) # type: ignore | |
except Exception as exc: | |
# Some corrupt tar files seem to produce this | |
# (specifically bad symlinks) | |
logger.warning( | |
'In the tar file %s the member %s is invalid: %s', | |
filename, member.name, exc, | |
) | |
continue | |
else: | |
try: | |
fp = tar.extractfile(member) | |
except (KeyError, AttributeError) as exc: | |
# Some corrupt tar files seem to produce this | |
# (specifically bad symlinks) | |
logger.warning( | |
'In the tar file %s the member %s is invalid: %s', | |
filename, member.name, exc, | |
) | |
continue | |
ensure_dir(os.path.dirname(path)) | |
with open(path, 'wb') as destfp: | |
shutil.copyfileobj(fp, destfp) | |
fp.close() | |
# Update the timestamp (useful for cython compiled files) | |
# https://github.com/python/typeshed/issues/2673 | |
tar.utime(member, path) # type: ignore | |
# member have any execute permissions for user/group/world? | |
if member.mode & 0o111: | |
# make dest file have execute for user/group/world | |
# no-op on windows per python docs | |
os.chmod(path, (0o777 - current_umask() | 0o111)) | |
finally: | |
tar.close() | |
def unpack_file( | |
filename, # type: str | |
location, # type: str | |
content_type=None, # type: Optional[str] | |
): | |
# type: (...) -> None | |
filename = os.path.realpath(filename) | |
if ( | |
content_type == 'application/zip' or | |
filename.lower().endswith(ZIP_EXTENSIONS) or | |
zipfile.is_zipfile(filename) | |
): | |
unzip_file( | |
filename, | |
location, | |
flatten=not filename.endswith('.whl') | |
) | |
elif ( | |
content_type == 'application/x-gzip' or | |
tarfile.is_tarfile(filename) or | |
filename.lower().endswith( | |
TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS | |
) | |
): | |
untar_file(filename, location) | |
else: | |
# FIXME: handle? | |
# FIXME: magic signatures? | |
logger.critical( | |
'Cannot unpack file %s (downloaded from %s, content-type: %s); ' | |
'cannot detect archive format', | |
filename, location, content_type, | |
) | |
raise InstallationError( | |
'Cannot determine archive format of {}'.format(location) | |
) |
import os | |
import sys | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._vendor.six.moves.urllib import request as urllib_request | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Text, Union | |
def get_url_scheme(url): | |
# type: (Union[str, Text]) -> Optional[Text] | |
if ':' not in url: | |
return None | |
return url.split(':', 1)[0].lower() | |
def path_to_url(path): | |
# type: (Union[str, Text]) -> str | |
""" | |
Convert a path to a file: URL. The path will be made absolute and have | |
quoted path parts. | |
""" | |
path = os.path.normpath(os.path.abspath(path)) | |
url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) | |
return url | |
def url_to_path(url): | |
# type: (str) -> str | |
""" | |
Convert a file: URL to a path. | |
""" | |
assert url.startswith('file:'), ( | |
"You can only turn file: urls into filenames (not %r)" % url) | |
_, netloc, path, _, _ = urllib_parse.urlsplit(url) | |
if not netloc or netloc == 'localhost': | |
# According to RFC 8089, same as empty authority. | |
netloc = '' | |
elif sys.platform == 'win32': | |
# If we have a UNC path, prepend UNC share notation. | |
netloc = '\\\\' + netloc | |
else: | |
raise ValueError( | |
'non-local file URIs are not supported on this platform: %r' | |
% url | |
) | |
path = urllib_request.url2pathname(netloc + path) | |
return path |
from __future__ import absolute_import | |
import logging | |
import os | |
import re | |
import site | |
import sys | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from typing import List, Optional | |
logger = logging.getLogger(__name__) | |
_INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( | |
r"include-system-site-packages\s*=\s*(?P<value>true|false)" | |
) | |
def _running_under_venv(): | |
# type: () -> bool | |
"""Checks if sys.base_prefix and sys.prefix match. | |
This handles PEP 405 compliant virtual environments. | |
""" | |
return sys.prefix != getattr(sys, "base_prefix", sys.prefix) | |
def _running_under_regular_virtualenv(): | |
# type: () -> bool | |
"""Checks if sys.real_prefix is set. | |
This handles virtual environments created with pypa's virtualenv. | |
""" | |
# pypa/virtualenv case | |
return hasattr(sys, 'real_prefix') | |
def running_under_virtualenv(): | |
# type: () -> bool | |
"""Return True if we're running inside a virtualenv, False otherwise. | |
""" | |
return _running_under_venv() or _running_under_regular_virtualenv() | |
def _get_pyvenv_cfg_lines(): | |
# type: () -> Optional[List[str]] | |
"""Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines | |
Returns None, if it could not read/access the file. | |
""" | |
pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') | |
try: | |
with open(pyvenv_cfg_file) as f: | |
return f.read().splitlines() # avoids trailing newlines | |
except IOError: | |
return None | |
def _no_global_under_venv(): | |
# type: () -> bool | |
"""Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion | |
PEP 405 specifies that when system site-packages are not supposed to be | |
visible from a virtual environment, `pyvenv.cfg` must contain the following | |
line: | |
include-system-site-packages = false | |
Additionally, log a warning if accessing the file fails. | |
""" | |
cfg_lines = _get_pyvenv_cfg_lines() | |
if cfg_lines is None: | |
# We're not in a "sane" venv, so assume there is no system | |
# site-packages access (since that's PEP 405's default state). | |
logger.warning( | |
"Could not access 'pyvenv.cfg' despite a virtual environment " | |
"being active. Assuming global site-packages is not accessible " | |
"in this environment." | |
) | |
return True | |
for line in cfg_lines: | |
match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) | |
if match is not None and match.group('value') == 'false': | |
return True | |
return False | |
def _no_global_under_regular_virtualenv(): | |
# type: () -> bool | |
"""Check if "no-global-site-packages.txt" exists beside site.py | |
This mirrors logic in pypa/virtualenv for determining whether system | |
site-packages are visible in the virtual environment. | |
""" | |
site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) | |
no_global_site_packages_file = os.path.join( | |
site_mod_dir, 'no-global-site-packages.txt', | |
) | |
return os.path.exists(no_global_site_packages_file) | |
def virtualenv_no_global(): | |
# type: () -> bool | |
"""Returns a boolean, whether running in venv with no system site-packages. | |
""" | |
if _running_under_regular_virtualenv(): | |
return _no_global_under_regular_virtualenv() | |
if _running_under_venv(): | |
return _no_global_under_venv() | |
return False |
"""Support functions for working with wheel files. | |
""" | |
from __future__ import absolute_import | |
import logging | |
from email.parser import Parser | |
from zipfile import ZipFile | |
from pip._vendor.packaging.utils import canonicalize_name | |
from pip._vendor.pkg_resources import DistInfoDistribution | |
from pip._vendor.six import PY2, ensure_str | |
from pip._internal.exceptions import UnsupportedWheel | |
from pip._internal.utils.pkg_resources import DictMetadata | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
if MYPY_CHECK_RUNNING: | |
from email.message import Message | |
from typing import Dict, Tuple | |
from pip._vendor.pkg_resources import Distribution | |
if PY2: | |
from zipfile import BadZipfile as BadZipFile | |
else: | |
from zipfile import BadZipFile | |
VERSION_COMPATIBLE = (1, 0) | |
logger = logging.getLogger(__name__) | |
class WheelMetadata(DictMetadata): | |
"""Metadata provider that maps metadata decoding exceptions to our | |
internal exception type. | |
""" | |
def __init__(self, metadata, wheel_name): | |
# type: (Dict[str, bytes], str) -> None | |
super(WheelMetadata, self).__init__(metadata) | |
self._wheel_name = wheel_name | |
def get_metadata(self, name): | |
# type: (str) -> str | |
try: | |
return super(WheelMetadata, self).get_metadata(name) | |
except UnicodeDecodeError as e: | |
# Augment the default error with the origin of the file. | |
raise UnsupportedWheel( | |
"Error decoding metadata for {}: {}".format( | |
self._wheel_name, e | |
) | |
) | |
def pkg_resources_distribution_for_wheel(wheel_zip, name, location): | |
# type: (ZipFile, str, str) -> Distribution | |
"""Get a pkg_resources distribution given a wheel. | |
:raises UnsupportedWheel: on any errors | |
""" | |
info_dir, _ = parse_wheel(wheel_zip, name) | |
metadata_files = [ | |
p for p in wheel_zip.namelist() if p.startswith("{}/".format(info_dir)) | |
] | |
metadata_text = {} # type: Dict[str, bytes] | |
for path in metadata_files: | |
# If a flag is set, namelist entries may be unicode in Python 2. | |
# We coerce them to native str type to match the types used in the rest | |
# of the code. This cannot fail because unicode can always be encoded | |
# with UTF-8. | |
full_path = ensure_str(path) | |
_, metadata_name = full_path.split("/", 1) | |
try: | |
metadata_text[metadata_name] = read_wheel_metadata_file( | |
wheel_zip, full_path | |
) | |
except UnsupportedWheel as e: | |
raise UnsupportedWheel( | |
"{} has an invalid wheel, {}".format(name, str(e)) | |
) | |
metadata = WheelMetadata(metadata_text, location) | |
return DistInfoDistribution( | |
location=location, metadata=metadata, project_name=name | |
) | |
def parse_wheel(wheel_zip, name): | |
# type: (ZipFile, str) -> Tuple[str, Message] | |
"""Extract information from the provided wheel, ensuring it meets basic | |
standards. | |
Returns the name of the .dist-info directory and the parsed WHEEL metadata. | |
""" | |
try: | |
info_dir = wheel_dist_info_dir(wheel_zip, name) | |
metadata = wheel_metadata(wheel_zip, info_dir) | |
version = wheel_version(metadata) | |
except UnsupportedWheel as e: | |
raise UnsupportedWheel( | |
"{} has an invalid wheel, {}".format(name, str(e)) | |
) | |
check_compatibility(version, name) | |
return info_dir, metadata | |
def wheel_dist_info_dir(source, name): | |
# type: (ZipFile, str) -> str | |
"""Returns the name of the contained .dist-info directory. | |
Raises AssertionError or UnsupportedWheel if not found, >1 found, or | |
it doesn't match the provided name. | |
""" | |
# Zip file path separators must be / | |
subdirs = list(set(p.split("/")[0] for p in source.namelist())) | |
info_dirs = [s for s in subdirs if s.endswith('.dist-info')] | |
if not info_dirs: | |
raise UnsupportedWheel(".dist-info directory not found") | |
if len(info_dirs) > 1: | |
raise UnsupportedWheel( | |
"multiple .dist-info directories found: {}".format( | |
", ".join(info_dirs) | |
) | |
) | |
info_dir = info_dirs[0] | |
info_dir_name = canonicalize_name(info_dir) | |
canonical_name = canonicalize_name(name) | |
if not info_dir_name.startswith(canonical_name): | |
raise UnsupportedWheel( | |
".dist-info directory {!r} does not start with {!r}".format( | |
info_dir, canonical_name | |
) | |
) | |
# Zip file paths can be unicode or str depending on the zip entry flags, | |
# so normalize it. | |
return ensure_str(info_dir) | |
def read_wheel_metadata_file(source, path): | |
# type: (ZipFile, str) -> bytes | |
try: | |
return source.read(path) | |
# BadZipFile for general corruption, KeyError for missing entry, | |
# and RuntimeError for password-protected files | |
except (BadZipFile, KeyError, RuntimeError) as e: | |
raise UnsupportedWheel( | |
"could not read {!r} file: {!r}".format(path, e) | |
) | |
def wheel_metadata(source, dist_info_dir): | |
# type: (ZipFile, str) -> Message | |
"""Return the WHEEL metadata of an extracted wheel, if possible. | |
Otherwise, raise UnsupportedWheel. | |
""" | |
path = "{}/WHEEL".format(dist_info_dir) | |
# Zip file path separators must be / | |
wheel_contents = read_wheel_metadata_file(source, path) | |
try: | |
wheel_text = ensure_str(wheel_contents) | |
except UnicodeDecodeError as e: | |
raise UnsupportedWheel("error decoding {!r}: {!r}".format(path, e)) | |
# FeedParser (used by Parser) does not raise any exceptions. The returned | |
# message may have .defects populated, but for backwards-compatibility we | |
# currently ignore them. | |
return Parser().parsestr(wheel_text) | |
def wheel_version(wheel_data): | |
# type: (Message) -> Tuple[int, ...] | |
"""Given WHEEL metadata, return the parsed Wheel-Version. | |
Otherwise, raise UnsupportedWheel. | |
""" | |
version_text = wheel_data["Wheel-Version"] | |
if version_text is None: | |
raise UnsupportedWheel("WHEEL is missing Wheel-Version") | |
version = version_text.strip() | |
try: | |
return tuple(map(int, version.split('.'))) | |
except ValueError: | |
raise UnsupportedWheel("invalid Wheel-Version: {!r}".format(version)) | |
def check_compatibility(version, name): | |
# type: (Tuple[int, ...], str) -> None | |
"""Raises errors or warns if called with an incompatible Wheel-Version. | |
Pip should refuse to install a Wheel-Version that's a major series | |
ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when | |
installing a version only minor version ahead (e.g 1.2 > 1.1). | |
version: a 2-tuple representing a Wheel-Version (Major, Minor) | |
name: name of wheel or package to raise exception about | |
:raises UnsupportedWheel: when an incompatible Wheel-Version is given | |
""" | |
if version[0] > VERSION_COMPATIBLE[0]: | |
raise UnsupportedWheel( | |
"%s's Wheel-Version (%s) is not compatible with this version " | |
"of pip" % (name, '.'.join(map(str, version))) | |
) | |
elif version > VERSION_COMPATIBLE: | |
logger.warning( | |
'Installing from a newer Wheel-Version (%s)', | |
'.'.join(map(str, version)), | |
) |
# Expose a limited set of classes and functions so callers outside of | |
# the vcs package don't need to import deeper than `pip._internal.vcs`. | |
# (The test directory and imports protected by MYPY_CHECK_RUNNING may | |
# still need to import from a vcs sub-package.) | |
# Import all vcs modules to register each VCS in the VcsSupport object. | |
import pip._internal.vcs.bazaar | |
import pip._internal.vcs.git | |
import pip._internal.vcs.mercurial | |
import pip._internal.vcs.subversion # noqa: F401 | |
from pip._internal.vcs.versioncontrol import ( # noqa: F401 | |
RemoteNotFoundError, | |
is_url, | |
make_vcs_requirement_url, | |
vcs, | |
) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.utils.misc import display_path, rmtree | |
from pip._internal.utils.subprocess import make_command | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url | |
from pip._internal.vcs.versioncontrol import VersionControl, vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Tuple | |
from pip._internal.utils.misc import HiddenText | |
from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions | |
logger = logging.getLogger(__name__) | |
class Bazaar(VersionControl): | |
name = 'bzr' | |
dirname = '.bzr' | |
repo_name = 'branch' | |
schemes = ( | |
'bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp', | |
'bzr+lp', | |
) | |
def __init__(self, *args, **kwargs): | |
super(Bazaar, self).__init__(*args, **kwargs) | |
# This is only needed for python <2.7.5 | |
# Register lp but do not expose as a scheme to support bzr+lp. | |
if getattr(urllib_parse, 'uses_fragment', None): | |
urllib_parse.uses_fragment.extend(['lp']) | |
@staticmethod | |
def get_base_rev_args(rev): | |
return ['-r', rev] | |
def export(self, location, url): | |
# type: (str, HiddenText) -> None | |
""" | |
Export the Bazaar repository at the url to the destination location | |
""" | |
# Remove the location to make sure Bazaar can export it correctly | |
if os.path.exists(location): | |
rmtree(location) | |
url, rev_options = self.get_url_rev_options(url) | |
self.run_command( | |
make_command('export', location, url, rev_options.to_args()), | |
show_stdout=False, | |
) | |
def fetch_new(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
rev_display = rev_options.to_display() | |
logger.info( | |
'Checking out %s%s to %s', | |
url, | |
rev_display, | |
display_path(dest), | |
) | |
cmd_args = ( | |
make_command('branch', '-q', rev_options.to_args(), url, dest) | |
) | |
self.run_command(cmd_args) | |
def switch(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
self.run_command(make_command('switch', url), cwd=dest) | |
def update(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
cmd_args = make_command('pull', '-q', rev_options.to_args()) | |
self.run_command(cmd_args, cwd=dest) | |
@classmethod | |
def get_url_rev_and_auth(cls, url): | |
# type: (str) -> Tuple[str, Optional[str], AuthInfo] | |
# hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it | |
url, rev, user_pass = super(Bazaar, cls).get_url_rev_and_auth(url) | |
if url.startswith('ssh://'): | |
url = 'bzr+' + url | |
return url, rev, user_pass | |
@classmethod | |
def get_remote_url(cls, location): | |
urls = cls.run_command(['info'], show_stdout=False, cwd=location) | |
for line in urls.splitlines(): | |
line = line.strip() | |
for x in ('checkout of branch: ', | |
'parent branch: '): | |
if line.startswith(x): | |
repo = line.split(x)[1] | |
if cls._is_local_repository(repo): | |
return path_to_url(repo) | |
return repo | |
return None | |
@classmethod | |
def get_revision(cls, location): | |
revision = cls.run_command( | |
['revno'], show_stdout=False, cwd=location, | |
) | |
return revision.splitlines()[-1] | |
@classmethod | |
def is_commit_id_equal(cls, dest, name): | |
"""Always assume the versions don't match""" | |
return False | |
vcs.register(Bazaar) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os.path | |
import re | |
from pip._vendor.packaging.version import parse as parse_version | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._vendor.six.moves.urllib import request as urllib_request | |
from pip._internal.exceptions import BadCommand | |
from pip._internal.utils.misc import display_path, hide_url | |
from pip._internal.utils.subprocess import make_command | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.vcs.versioncontrol import ( | |
RemoteNotFoundError, | |
VersionControl, | |
find_path_to_setup_from_repo_root, | |
vcs, | |
) | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Tuple | |
from pip._internal.utils.misc import HiddenText | |
from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions | |
urlsplit = urllib_parse.urlsplit | |
urlunsplit = urllib_parse.urlunsplit | |
logger = logging.getLogger(__name__) | |
HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') | |
def looks_like_hash(sha): | |
return bool(HASH_REGEX.match(sha)) | |
class Git(VersionControl): | |
name = 'git' | |
dirname = '.git' | |
repo_name = 'clone' | |
schemes = ( | |
'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', | |
) | |
# Prevent the user's environment variables from interfering with pip: | |
# https://github.com/pypa/pip/issues/1130 | |
unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') | |
default_arg_rev = 'HEAD' | |
@staticmethod | |
def get_base_rev_args(rev): | |
return [rev] | |
def is_immutable_rev_checkout(self, url, dest): | |
# type: (str, str) -> bool | |
_, rev_options = self.get_url_rev_options(hide_url(url)) | |
if not rev_options.rev: | |
return False | |
if not self.is_commit_id_equal(dest, rev_options.rev): | |
# the current commit is different from rev, | |
# which means rev was something else than a commit hash | |
return False | |
# return False in the rare case rev is both a commit hash | |
# and a tag or a branch; we don't want to cache in that case | |
# because that branch/tag could point to something else in the future | |
is_tag_or_branch = bool( | |
self.get_revision_sha(dest, rev_options.rev)[0] | |
) | |
return not is_tag_or_branch | |
def get_git_version(self): | |
VERSION_PFX = 'git version ' | |
version = self.run_command(['version'], show_stdout=False) | |
if version.startswith(VERSION_PFX): | |
version = version[len(VERSION_PFX):].split()[0] | |
else: | |
version = '' | |
# get first 3 positions of the git version because | |
# on windows it is x.y.z.windows.t, and this parses as | |
# LegacyVersion which always smaller than a Version. | |
version = '.'.join(version.split('.')[:3]) | |
return parse_version(version) | |
@classmethod | |
def get_current_branch(cls, location): | |
""" | |
Return the current branch, or None if HEAD isn't at a branch | |
(e.g. detached HEAD). | |
""" | |
# git-symbolic-ref exits with empty stdout if "HEAD" is a detached | |
# HEAD rather than a symbolic ref. In addition, the -q causes the | |
# command to exit with status code 1 instead of 128 in this case | |
# and to suppress the message to stderr. | |
args = ['symbolic-ref', '-q', 'HEAD'] | |
output = cls.run_command( | |
args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, | |
) | |
ref = output.strip() | |
if ref.startswith('refs/heads/'): | |
return ref[len('refs/heads/'):] | |
return None | |
def export(self, location, url): | |
# type: (str, HiddenText) -> None | |
"""Export the Git repository at the url to the destination location""" | |
if not location.endswith('/'): | |
location = location + '/' | |
with TempDirectory(kind="export") as temp_dir: | |
self.unpack(temp_dir.path, url=url) | |
self.run_command( | |
['checkout-index', '-a', '-f', '--prefix', location], | |
show_stdout=False, cwd=temp_dir.path | |
) | |
@classmethod | |
def get_revision_sha(cls, dest, rev): | |
""" | |
Return (sha_or_none, is_branch), where sha_or_none is a commit hash | |
if the revision names a remote branch or tag, otherwise None. | |
Args: | |
dest: the repository directory. | |
rev: the revision name. | |
""" | |
# Pass rev to pre-filter the list. | |
output = cls.run_command(['show-ref', rev], cwd=dest, | |
show_stdout=False, on_returncode='ignore') | |
refs = {} | |
for line in output.strip().splitlines(): | |
try: | |
sha, ref = line.split() | |
except ValueError: | |
# Include the offending line to simplify troubleshooting if | |
# this error ever occurs. | |
raise ValueError('unexpected show-ref line: {!r}'.format(line)) | |
refs[ref] = sha | |
branch_ref = 'refs/remotes/origin/{}'.format(rev) | |
tag_ref = 'refs/tags/{}'.format(rev) | |
sha = refs.get(branch_ref) | |
if sha is not None: | |
return (sha, True) | |
sha = refs.get(tag_ref) | |
return (sha, False) | |
@classmethod | |
def resolve_revision(cls, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> RevOptions | |
""" | |
Resolve a revision to a new RevOptions object with the SHA1 of the | |
branch, tag, or ref if found. | |
Args: | |
rev_options: a RevOptions object. | |
""" | |
rev = rev_options.arg_rev | |
# The arg_rev property's implementation for Git ensures that the | |
# rev return value is always non-None. | |
assert rev is not None | |
sha, is_branch = cls.get_revision_sha(dest, rev) | |
if sha is not None: | |
rev_options = rev_options.make_new(sha) | |
rev_options.branch_name = rev if is_branch else None | |
return rev_options | |
# Do not show a warning for the common case of something that has | |
# the form of a Git commit hash. | |
if not looks_like_hash(rev): | |
logger.warning( | |
"Did not find branch or tag '%s', assuming revision or ref.", | |
rev, | |
) | |
if not rev.startswith('refs/'): | |
return rev_options | |
# If it looks like a ref, we have to fetch it explicitly. | |
cls.run_command( | |
make_command('fetch', '-q', url, rev_options.to_args()), | |
cwd=dest, | |
) | |
# Change the revision to the SHA of the ref we fetched | |
sha = cls.get_revision(dest, rev='FETCH_HEAD') | |
rev_options = rev_options.make_new(sha) | |
return rev_options | |
@classmethod | |
def is_commit_id_equal(cls, dest, name): | |
""" | |
Return whether the current commit hash equals the given name. | |
Args: | |
dest: the repository directory. | |
name: a string name. | |
""" | |
if not name: | |
# Then avoid an unnecessary subprocess call. | |
return False | |
return cls.get_revision(dest) == name | |
def fetch_new(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
rev_display = rev_options.to_display() | |
logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) | |
self.run_command(make_command('clone', '-q', url, dest)) | |
if rev_options.rev: | |
# Then a specific revision was requested. | |
rev_options = self.resolve_revision(dest, url, rev_options) | |
branch_name = getattr(rev_options, 'branch_name', None) | |
if branch_name is None: | |
# Only do a checkout if the current commit id doesn't match | |
# the requested revision. | |
if not self.is_commit_id_equal(dest, rev_options.rev): | |
cmd_args = make_command( | |
'checkout', '-q', rev_options.to_args(), | |
) | |
self.run_command(cmd_args, cwd=dest) | |
elif self.get_current_branch(dest) != branch_name: | |
# Then a specific branch was requested, and that branch | |
# is not yet checked out. | |
track_branch = 'origin/{}'.format(branch_name) | |
cmd_args = [ | |
'checkout', '-b', branch_name, '--track', track_branch, | |
] | |
self.run_command(cmd_args, cwd=dest) | |
#: repo may contain submodules | |
self.update_submodules(dest) | |
def switch(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
self.run_command( | |
make_command('config', 'remote.origin.url', url), | |
cwd=dest, | |
) | |
cmd_args = make_command('checkout', '-q', rev_options.to_args()) | |
self.run_command(cmd_args, cwd=dest) | |
self.update_submodules(dest) | |
def update(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
# First fetch changes from the default remote | |
if self.get_git_version() >= parse_version('1.9.0'): | |
# fetch tags in addition to everything else | |
self.run_command(['fetch', '-q', '--tags'], cwd=dest) | |
else: | |
self.run_command(['fetch', '-q'], cwd=dest) | |
# Then reset to wanted revision (maybe even origin/master) | |
rev_options = self.resolve_revision(dest, url, rev_options) | |
cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) | |
self.run_command(cmd_args, cwd=dest) | |
#: update submodules | |
self.update_submodules(dest) | |
@classmethod | |
def get_remote_url(cls, location): | |
""" | |
Return URL of the first remote encountered. | |
Raises RemoteNotFoundError if the repository does not have a remote | |
url configured. | |
""" | |
# We need to pass 1 for extra_ok_returncodes since the command | |
# exits with return code 1 if there are no matching lines. | |
stdout = cls.run_command( | |
['config', '--get-regexp', r'remote\..*\.url'], | |
extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, | |
) | |
remotes = stdout.splitlines() | |
try: | |
found_remote = remotes[0] | |
except IndexError: | |
raise RemoteNotFoundError | |
for remote in remotes: | |
if remote.startswith('remote.origin.url '): | |
found_remote = remote | |
break | |
url = found_remote.split(' ')[1] | |
return url.strip() | |
@classmethod | |
def get_revision(cls, location, rev=None): | |
if rev is None: | |
rev = 'HEAD' | |
current_rev = cls.run_command( | |
['rev-parse', rev], show_stdout=False, cwd=location, | |
) | |
return current_rev.strip() | |
@classmethod | |
def get_subdirectory(cls, location): | |
""" | |
Return the path to setup.py, relative to the repo root. | |
Return None if setup.py is in the repo root. | |
""" | |
# find the repo root | |
git_dir = cls.run_command( | |
['rev-parse', '--git-dir'], | |
show_stdout=False, cwd=location).strip() | |
if not os.path.isabs(git_dir): | |
git_dir = os.path.join(location, git_dir) | |
repo_root = os.path.abspath(os.path.join(git_dir, '..')) | |
return find_path_to_setup_from_repo_root(location, repo_root) | |
@classmethod | |
def get_url_rev_and_auth(cls, url): | |
# type: (str) -> Tuple[str, Optional[str], AuthInfo] | |
""" | |
Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. | |
That's required because although they use SSH they sometimes don't | |
work with a ssh:// scheme (e.g. GitHub). But we need a scheme for | |
parsing. Hence we remove it again afterwards and return it as a stub. | |
""" | |
# Works around an apparent Git bug | |
# (see https://article.gmane.org/gmane.comp.version-control.git/146500) | |
scheme, netloc, path, query, fragment = urlsplit(url) | |
if scheme.endswith('file'): | |
initial_slashes = path[:-len(path.lstrip('/'))] | |
newpath = ( | |
initial_slashes + | |
urllib_request.url2pathname(path) | |
.replace('\\', '/').lstrip('/') | |
) | |
url = urlunsplit((scheme, netloc, newpath, query, fragment)) | |
after_plus = scheme.find('+') + 1 | |
url = scheme[:after_plus] + urlunsplit( | |
(scheme[after_plus:], netloc, newpath, query, fragment), | |
) | |
if '://' not in url: | |
assert 'file:' not in url | |
url = url.replace('git+', 'git+ssh://') | |
url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) | |
url = url.replace('ssh://', '') | |
else: | |
url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) | |
return url, rev, user_pass | |
@classmethod | |
def update_submodules(cls, location): | |
if not os.path.exists(os.path.join(location, '.gitmodules')): | |
return | |
cls.run_command( | |
['submodule', 'update', '--init', '--recursive', '-q'], | |
cwd=location, | |
) | |
@classmethod | |
def controls_location(cls, location): | |
if super(Git, cls).controls_location(location): | |
return True | |
try: | |
r = cls.run_command(['rev-parse'], | |
cwd=location, | |
show_stdout=False, | |
on_returncode='ignore', | |
log_failed_cmd=False) | |
return not r | |
except BadCommand: | |
logger.debug("could not determine if %s is under git control " | |
"because git is not available", location) | |
return False | |
vcs.register(Git) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
from pip._vendor.six.moves import configparser | |
from pip._internal.exceptions import BadCommand, InstallationError | |
from pip._internal.utils.misc import display_path | |
from pip._internal.utils.subprocess import make_command | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url | |
from pip._internal.vcs.versioncontrol import ( | |
VersionControl, | |
find_path_to_setup_from_repo_root, | |
vcs, | |
) | |
if MYPY_CHECK_RUNNING: | |
from pip._internal.utils.misc import HiddenText | |
from pip._internal.vcs.versioncontrol import RevOptions | |
logger = logging.getLogger(__name__) | |
class Mercurial(VersionControl): | |
name = 'hg' | |
dirname = '.hg' | |
repo_name = 'clone' | |
schemes = ( | |
'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', | |
) | |
@staticmethod | |
def get_base_rev_args(rev): | |
return [rev] | |
def export(self, location, url): | |
# type: (str, HiddenText) -> None | |
"""Export the Hg repository at the url to the destination location""" | |
with TempDirectory(kind="export") as temp_dir: | |
self.unpack(temp_dir.path, url=url) | |
self.run_command( | |
['archive', location], show_stdout=False, cwd=temp_dir.path | |
) | |
def fetch_new(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
rev_display = rev_options.to_display() | |
logger.info( | |
'Cloning hg %s%s to %s', | |
url, | |
rev_display, | |
display_path(dest), | |
) | |
self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) | |
self.run_command( | |
make_command('update', '-q', rev_options.to_args()), | |
cwd=dest, | |
) | |
def switch(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
repo_config = os.path.join(dest, self.dirname, 'hgrc') | |
config = configparser.RawConfigParser() | |
try: | |
config.read(repo_config) | |
config.set('paths', 'default', url.secret) | |
with open(repo_config, 'w') as config_file: | |
config.write(config_file) | |
except (OSError, configparser.NoSectionError) as exc: | |
logger.warning( | |
'Could not switch Mercurial repository to %s: %s', url, exc, | |
) | |
else: | |
cmd_args = make_command('update', '-q', rev_options.to_args()) | |
self.run_command(cmd_args, cwd=dest) | |
def update(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
self.run_command(['pull', '-q'], cwd=dest) | |
cmd_args = make_command('update', '-q', rev_options.to_args()) | |
self.run_command(cmd_args, cwd=dest) | |
@classmethod | |
def get_remote_url(cls, location): | |
url = cls.run_command( | |
['showconfig', 'paths.default'], | |
show_stdout=False, cwd=location).strip() | |
if cls._is_local_repository(url): | |
url = path_to_url(url) | |
return url.strip() | |
@classmethod | |
def get_revision(cls, location): | |
""" | |
Return the repository-local changeset revision number, as an integer. | |
""" | |
current_revision = cls.run_command( | |
['parents', '--template={rev}'], | |
show_stdout=False, cwd=location).strip() | |
return current_revision | |
@classmethod | |
def get_requirement_revision(cls, location): | |
""" | |
Return the changeset identification hash, as a 40-character | |
hexadecimal string | |
""" | |
current_rev_hash = cls.run_command( | |
['parents', '--template={node}'], | |
show_stdout=False, cwd=location).strip() | |
return current_rev_hash | |
@classmethod | |
def is_commit_id_equal(cls, dest, name): | |
"""Always assume the versions don't match""" | |
return False | |
@classmethod | |
def get_subdirectory(cls, location): | |
""" | |
Return the path to setup.py, relative to the repo root. | |
Return None if setup.py is in the repo root. | |
""" | |
# find the repo root | |
repo_root = cls.run_command( | |
['root'], show_stdout=False, cwd=location).strip() | |
if not os.path.isabs(repo_root): | |
repo_root = os.path.abspath(os.path.join(location, repo_root)) | |
return find_path_to_setup_from_repo_root(location, repo_root) | |
@classmethod | |
def controls_location(cls, location): | |
if super(Mercurial, cls).controls_location(location): | |
return True | |
try: | |
cls.run_command( | |
['identify'], | |
cwd=location, | |
show_stdout=False, | |
on_returncode='raise', | |
log_failed_cmd=False) | |
return True | |
except (BadCommand, InstallationError): | |
return False | |
vcs.register(Mercurial) |
# The following comment should be removed at some point in the future. | |
# mypy: disallow-untyped-defs=False | |
from __future__ import absolute_import | |
import logging | |
import os | |
import re | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import ( | |
display_path, | |
is_console_interactive, | |
rmtree, | |
split_auth_from_netloc, | |
) | |
from pip._internal.utils.subprocess import make_command | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.vcs.versioncontrol import VersionControl, vcs | |
_svn_xml_url_re = re.compile('url="([^"]+)"') | |
_svn_rev_re = re.compile(r'committed-rev="(\d+)"') | |
_svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') | |
_svn_info_xml_url_re = re.compile(r'<url>(.*)</url>') | |
if MYPY_CHECK_RUNNING: | |
from typing import Optional, Tuple | |
from pip._internal.utils.subprocess import CommandArgs | |
from pip._internal.utils.misc import HiddenText | |
from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions | |
logger = logging.getLogger(__name__) | |
class Subversion(VersionControl): | |
name = 'svn' | |
dirname = '.svn' | |
repo_name = 'checkout' | |
schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn') | |
@classmethod | |
def should_add_vcs_url_prefix(cls, remote_url): | |
return True | |
@staticmethod | |
def get_base_rev_args(rev): | |
return ['-r', rev] | |
@classmethod | |
def get_revision(cls, location): | |
""" | |
Return the maximum revision for all files under a given location | |
""" | |
# Note: taken from setuptools.command.egg_info | |
revision = 0 | |
for base, dirs, files in os.walk(location): | |
if cls.dirname not in dirs: | |
dirs[:] = [] | |
continue # no sense walking uncontrolled subdirs | |
dirs.remove(cls.dirname) | |
entries_fn = os.path.join(base, cls.dirname, 'entries') | |
if not os.path.exists(entries_fn): | |
# FIXME: should we warn? | |
continue | |
dirurl, localrev = cls._get_svn_url_rev(base) | |
if base == location: | |
base = dirurl + '/' # save the root url | |
elif not dirurl or not dirurl.startswith(base): | |
dirs[:] = [] | |
continue # not part of the same svn tree, skip it | |
revision = max(revision, localrev) | |
return revision | |
@classmethod | |
def get_netloc_and_auth(cls, netloc, scheme): | |
""" | |
This override allows the auth information to be passed to svn via the | |
--username and --password options instead of via the URL. | |
""" | |
if scheme == 'ssh': | |
# The --username and --password options can't be used for | |
# svn+ssh URLs, so keep the auth information in the URL. | |
return super(Subversion, cls).get_netloc_and_auth(netloc, scheme) | |
return split_auth_from_netloc(netloc) | |
@classmethod | |
def get_url_rev_and_auth(cls, url): | |
# type: (str) -> Tuple[str, Optional[str], AuthInfo] | |
# hotfix the URL scheme after removing svn+ from svn+ssh:// readd it | |
url, rev, user_pass = super(Subversion, cls).get_url_rev_and_auth(url) | |
if url.startswith('ssh://'): | |
url = 'svn+' + url | |
return url, rev, user_pass | |
@staticmethod | |
def make_rev_args(username, password): | |
# type: (Optional[str], Optional[HiddenText]) -> CommandArgs | |
extra_args = [] # type: CommandArgs | |
if username: | |
extra_args += ['--username', username] | |
if password: | |
extra_args += ['--password', password] | |
return extra_args | |
@classmethod | |
def get_remote_url(cls, location): | |
# In cases where the source is in a subdirectory, not alongside | |
# setup.py we have to look up in the location until we find a real | |
# setup.py | |
orig_location = location | |
while not os.path.exists(os.path.join(location, 'setup.py')): | |
last_location = location | |
location = os.path.dirname(location) | |
if location == last_location: | |
# We've traversed up to the root of the filesystem without | |
# finding setup.py | |
logger.warning( | |
"Could not find setup.py for directory %s (tried all " | |
"parent directories)", | |
orig_location, | |
) | |
return None | |
return cls._get_svn_url_rev(location)[0] | |
@classmethod | |
def _get_svn_url_rev(cls, location): | |
from pip._internal.exceptions import InstallationError | |
entries_path = os.path.join(location, cls.dirname, 'entries') | |
if os.path.exists(entries_path): | |
with open(entries_path) as f: | |
data = f.read() | |
else: # subversion >= 1.7 does not have the 'entries' file | |
data = '' | |
if (data.startswith('8') or | |
data.startswith('9') or | |
data.startswith('10')): | |
data = list(map(str.splitlines, data.split('\n\x0c\n'))) | |
del data[0][0] # get rid of the '8' | |
url = data[0][3] | |
revs = [int(d[9]) for d in data if len(d) > 9 and d[9]] + [0] | |
elif data.startswith('<?xml'): | |
match = _svn_xml_url_re.search(data) | |
if not match: | |
raise ValueError('Badly formatted data: %r' % data) | |
url = match.group(1) # get repository URL | |
revs = [int(m.group(1)) for m in _svn_rev_re.finditer(data)] + [0] | |
else: | |
try: | |
# subversion >= 1.7 | |
# Note that using get_remote_call_options is not necessary here | |
# because `svn info` is being run against a local directory. | |
# We don't need to worry about making sure interactive mode | |
# is being used to prompt for passwords, because passwords | |
# are only potentially needed for remote server requests. | |
xml = cls.run_command( | |
['info', '--xml', location], | |
show_stdout=False, | |
) | |
url = _svn_info_xml_url_re.search(xml).group(1) | |
revs = [ | |
int(m.group(1)) for m in _svn_info_xml_rev_re.finditer(xml) | |
] | |
except InstallationError: | |
url, revs = None, [] | |
if revs: | |
rev = max(revs) | |
else: | |
rev = 0 | |
return url, rev | |
@classmethod | |
def is_commit_id_equal(cls, dest, name): | |
"""Always assume the versions don't match""" | |
return False | |
def __init__(self, use_interactive=None): | |
# type: (bool) -> None | |
if use_interactive is None: | |
use_interactive = is_console_interactive() | |
self.use_interactive = use_interactive | |
# This member is used to cache the fetched version of the current | |
# ``svn`` client. | |
# Special value definitions: | |
# None: Not evaluated yet. | |
# Empty tuple: Could not parse version. | |
self._vcs_version = None # type: Optional[Tuple[int, ...]] | |
super(Subversion, self).__init__() | |
def call_vcs_version(self): | |
# type: () -> Tuple[int, ...] | |
"""Query the version of the currently installed Subversion client. | |
:return: A tuple containing the parts of the version information or | |
``()`` if the version returned from ``svn`` could not be parsed. | |
:raises: BadCommand: If ``svn`` is not installed. | |
""" | |
# Example versions: | |
# svn, version 1.10.3 (r1842928) | |
# compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0 | |
# svn, version 1.7.14 (r1542130) | |
# compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu | |
version_prefix = 'svn, version ' | |
version = self.run_command(['--version'], show_stdout=False) | |
if not version.startswith(version_prefix): | |
return () | |
version = version[len(version_prefix):].split()[0] | |
version_list = version.split('.') | |
try: | |
parsed_version = tuple(map(int, version_list)) | |
except ValueError: | |
return () | |
return parsed_version | |
def get_vcs_version(self): | |
# type: () -> Tuple[int, ...] | |
"""Return the version of the currently installed Subversion client. | |
If the version of the Subversion client has already been queried, | |
a cached value will be used. | |
:return: A tuple containing the parts of the version information or | |
``()`` if the version returned from ``svn`` could not be parsed. | |
:raises: BadCommand: If ``svn`` is not installed. | |
""" | |
if self._vcs_version is not None: | |
# Use cached version, if available. | |
# If parsing the version failed previously (empty tuple), | |
# do not attempt to parse it again. | |
return self._vcs_version | |
vcs_version = self.call_vcs_version() | |
self._vcs_version = vcs_version | |
return vcs_version | |
def get_remote_call_options(self): | |
# type: () -> CommandArgs | |
"""Return options to be used on calls to Subversion that contact the server. | |
These options are applicable for the following ``svn`` subcommands used | |
in this class. | |
- checkout | |
- export | |
- switch | |
- update | |
:return: A list of command line arguments to pass to ``svn``. | |
""" | |
if not self.use_interactive: | |
# --non-interactive switch is available since Subversion 0.14.4. | |
# Subversion < 1.8 runs in interactive mode by default. | |
return ['--non-interactive'] | |
svn_version = self.get_vcs_version() | |
# By default, Subversion >= 1.8 runs in non-interactive mode if | |
# stdin is not a TTY. Since that is how pip invokes SVN, in | |
# call_subprocess(), pip must pass --force-interactive to ensure | |
# the user can be prompted for a password, if required. | |
# SVN added the --force-interactive option in SVN 1.8. Since | |
# e.g. RHEL/CentOS 7, which is supported until 2024, ships with | |
# SVN 1.7, pip should continue to support SVN 1.7. Therefore, pip | |
# can't safely add the option if the SVN version is < 1.8 (or unknown). | |
if svn_version >= (1, 8): | |
return ['--force-interactive'] | |
return [] | |
def export(self, location, url): | |
# type: (str, HiddenText) -> None | |
"""Export the svn repository at the url to the destination location""" | |
url, rev_options = self.get_url_rev_options(url) | |
logger.info('Exporting svn repository %s to %s', url, location) | |
with indent_log(): | |
if os.path.exists(location): | |
# Subversion doesn't like to check out over an existing | |
# directory --force fixes this, but was only added in svn 1.5 | |
rmtree(location) | |
cmd_args = make_command( | |
'export', self.get_remote_call_options(), | |
rev_options.to_args(), url, location, | |
) | |
self.run_command(cmd_args, show_stdout=False) | |
def fetch_new(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
rev_display = rev_options.to_display() | |
logger.info( | |
'Checking out %s%s to %s', | |
url, | |
rev_display, | |
display_path(dest), | |
) | |
cmd_args = make_command( | |
'checkout', '-q', self.get_remote_call_options(), | |
rev_options.to_args(), url, dest, | |
) | |
self.run_command(cmd_args) | |
def switch(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
cmd_args = make_command( | |
'switch', self.get_remote_call_options(), rev_options.to_args(), | |
url, dest, | |
) | |
self.run_command(cmd_args) | |
def update(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
cmd_args = make_command( | |
'update', self.get_remote_call_options(), rev_options.to_args(), | |
dest, | |
) | |
self.run_command(cmd_args) | |
vcs.register(Subversion) |
"""Handles all VCS (version control) support""" | |
from __future__ import absolute_import | |
import errno | |
import logging | |
import os | |
import shutil | |
import sys | |
from pip._vendor import pkg_resources | |
from pip._vendor.six.moves.urllib import parse as urllib_parse | |
from pip._internal.exceptions import BadCommand | |
from pip._internal.utils.compat import samefile | |
from pip._internal.utils.misc import ( | |
ask_path_exists, | |
backup_dir, | |
display_path, | |
hide_url, | |
hide_value, | |
rmtree, | |
) | |
from pip._internal.utils.subprocess import call_subprocess, make_command | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import get_url_scheme | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Dict, Iterable, Iterator, List, Mapping, Optional, Text, Tuple, | |
Type, Union | |
) | |
from pip._internal.utils.ui import SpinnerInterface | |
from pip._internal.utils.misc import HiddenText | |
from pip._internal.utils.subprocess import CommandArgs | |
AuthInfo = Tuple[Optional[str], Optional[str]] | |
__all__ = ['vcs'] | |
logger = logging.getLogger(__name__) | |
def is_url(name): | |
# type: (Union[str, Text]) -> bool | |
""" | |
Return true if the name looks like a URL. | |
""" | |
scheme = get_url_scheme(name) | |
if scheme is None: | |
return False | |
return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes | |
def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): | |
# type: (str, str, str, Optional[str]) -> str | |
""" | |
Return the URL for a VCS requirement. | |
Args: | |
repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). | |
project_name: the (unescaped) project name. | |
""" | |
egg_project_name = pkg_resources.to_filename(project_name) | |
req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) | |
if subdir: | |
req += '&subdirectory={}'.format(subdir) | |
return req | |
def find_path_to_setup_from_repo_root(location, repo_root): | |
# type: (str, str) -> Optional[str] | |
""" | |
Find the path to `setup.py` by searching up the filesystem from `location`. | |
Return the path to `setup.py` relative to `repo_root`. | |
Return None if `setup.py` is in `repo_root` or cannot be found. | |
""" | |
# find setup.py | |
orig_location = location | |
while not os.path.exists(os.path.join(location, 'setup.py')): | |
last_location = location | |
location = os.path.dirname(location) | |
if location == last_location: | |
# We've traversed up to the root of the filesystem without | |
# finding setup.py | |
logger.warning( | |
"Could not find setup.py for directory %s (tried all " | |
"parent directories)", | |
orig_location, | |
) | |
return None | |
if samefile(repo_root, location): | |
return None | |
return os.path.relpath(location, repo_root) | |
class RemoteNotFoundError(Exception): | |
pass | |
class RevOptions(object): | |
""" | |
Encapsulates a VCS-specific revision to install, along with any VCS | |
install options. | |
Instances of this class should be treated as if immutable. | |
""" | |
def __init__( | |
self, | |
vc_class, # type: Type[VersionControl] | |
rev=None, # type: Optional[str] | |
extra_args=None, # type: Optional[CommandArgs] | |
): | |
# type: (...) -> None | |
""" | |
Args: | |
vc_class: a VersionControl subclass. | |
rev: the name of the revision to install. | |
extra_args: a list of extra options. | |
""" | |
if extra_args is None: | |
extra_args = [] | |
self.extra_args = extra_args | |
self.rev = rev | |
self.vc_class = vc_class | |
self.branch_name = None # type: Optional[str] | |
def __repr__(self): | |
# type: () -> str | |
return '<RevOptions {}: rev={!r}>'.format(self.vc_class.name, self.rev) | |
@property | |
def arg_rev(self): | |
# type: () -> Optional[str] | |
if self.rev is None: | |
return self.vc_class.default_arg_rev | |
return self.rev | |
def to_args(self): | |
# type: () -> CommandArgs | |
""" | |
Return the VCS-specific command arguments. | |
""" | |
args = [] # type: CommandArgs | |
rev = self.arg_rev | |
if rev is not None: | |
args += self.vc_class.get_base_rev_args(rev) | |
args += self.extra_args | |
return args | |
def to_display(self): | |
# type: () -> str | |
if not self.rev: | |
return '' | |
return ' (to revision {})'.format(self.rev) | |
def make_new(self, rev): | |
# type: (str) -> RevOptions | |
""" | |
Make a copy of the current instance, but with a new rev. | |
Args: | |
rev: the name of the revision for the new object. | |
""" | |
return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) | |
class VcsSupport(object): | |
_registry = {} # type: Dict[str, VersionControl] | |
schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] | |
def __init__(self): | |
# type: () -> None | |
# Register more schemes with urlparse for various version control | |
# systems | |
urllib_parse.uses_netloc.extend(self.schemes) | |
# Python >= 2.7.4, 3.3 doesn't have uses_fragment | |
if getattr(urllib_parse, 'uses_fragment', None): | |
urllib_parse.uses_fragment.extend(self.schemes) | |
super(VcsSupport, self).__init__() | |
def __iter__(self): | |
# type: () -> Iterator[str] | |
return self._registry.__iter__() | |
@property | |
def backends(self): | |
# type: () -> List[VersionControl] | |
return list(self._registry.values()) | |
@property | |
def dirnames(self): | |
# type: () -> List[str] | |
return [backend.dirname for backend in self.backends] | |
@property | |
def all_schemes(self): | |
# type: () -> List[str] | |
schemes = [] # type: List[str] | |
for backend in self.backends: | |
schemes.extend(backend.schemes) | |
return schemes | |
def register(self, cls): | |
# type: (Type[VersionControl]) -> None | |
if not hasattr(cls, 'name'): | |
logger.warning('Cannot register VCS %s', cls.__name__) | |
return | |
if cls.name not in self._registry: | |
self._registry[cls.name] = cls() | |
logger.debug('Registered VCS backend: %s', cls.name) | |
def unregister(self, name): | |
# type: (str) -> None | |
if name in self._registry: | |
del self._registry[name] | |
def get_backend_for_dir(self, location): | |
# type: (str) -> Optional[VersionControl] | |
""" | |
Return a VersionControl object if a repository of that type is found | |
at the given directory. | |
""" | |
for vcs_backend in self._registry.values(): | |
if vcs_backend.controls_location(location): | |
logger.debug('Determine that %s uses VCS: %s', | |
location, vcs_backend.name) | |
return vcs_backend | |
return None | |
def get_backend_for_scheme(self, scheme): | |
# type: (str) -> Optional[VersionControl] | |
""" | |
Return a VersionControl object or None. | |
""" | |
for vcs_backend in self._registry.values(): | |
if scheme in vcs_backend.schemes: | |
return vcs_backend | |
return None | |
def get_backend(self, name): | |
# type: (str) -> Optional[VersionControl] | |
""" | |
Return a VersionControl object or None. | |
""" | |
name = name.lower() | |
return self._registry.get(name) | |
vcs = VcsSupport() | |
class VersionControl(object): | |
name = '' | |
dirname = '' | |
repo_name = '' | |
# List of supported schemes for this Version Control | |
schemes = () # type: Tuple[str, ...] | |
# Iterable of environment variable names to pass to call_subprocess(). | |
unset_environ = () # type: Tuple[str, ...] | |
default_arg_rev = None # type: Optional[str] | |
@classmethod | |
def should_add_vcs_url_prefix(cls, remote_url): | |
# type: (str) -> bool | |
""" | |
Return whether the vcs prefix (e.g. "git+") should be added to a | |
repository's remote url when used in a requirement. | |
""" | |
return not remote_url.lower().startswith('{}:'.format(cls.name)) | |
@classmethod | |
def get_subdirectory(cls, location): | |
# type: (str) -> Optional[str] | |
""" | |
Return the path to setup.py, relative to the repo root. | |
Return None if setup.py is in the repo root. | |
""" | |
return None | |
@classmethod | |
def get_requirement_revision(cls, repo_dir): | |
# type: (str) -> str | |
""" | |
Return the revision string that should be used in a requirement. | |
""" | |
return cls.get_revision(repo_dir) | |
@classmethod | |
def get_src_requirement(cls, repo_dir, project_name): | |
# type: (str, str) -> Optional[str] | |
""" | |
Return the requirement string to use to redownload the files | |
currently at the given repository directory. | |
Args: | |
project_name: the (unescaped) project name. | |
The return value has a form similar to the following: | |
{repository_url}@{revision}#egg={project_name} | |
""" | |
repo_url = cls.get_remote_url(repo_dir) | |
if repo_url is None: | |
return None | |
if cls.should_add_vcs_url_prefix(repo_url): | |
repo_url = '{}+{}'.format(cls.name, repo_url) | |
revision = cls.get_requirement_revision(repo_dir) | |
subdir = cls.get_subdirectory(repo_dir) | |
req = make_vcs_requirement_url(repo_url, revision, project_name, | |
subdir=subdir) | |
return req | |
@staticmethod | |
def get_base_rev_args(rev): | |
# type: (str) -> List[str] | |
""" | |
Return the base revision arguments for a vcs command. | |
Args: | |
rev: the name of a revision to install. Cannot be None. | |
""" | |
raise NotImplementedError | |
def is_immutable_rev_checkout(self, url, dest): | |
# type: (str, str) -> bool | |
""" | |
Return true if the commit hash checked out at dest matches | |
the revision in url. | |
Always return False, if the VCS does not support immutable commit | |
hashes. | |
This method does not check if there are local uncommitted changes | |
in dest after checkout, as pip currently has no use case for that. | |
""" | |
return False | |
@classmethod | |
def make_rev_options(cls, rev=None, extra_args=None): | |
# type: (Optional[str], Optional[CommandArgs]) -> RevOptions | |
""" | |
Return a RevOptions object. | |
Args: | |
rev: the name of a revision to install. | |
extra_args: a list of extra options. | |
""" | |
return RevOptions(cls, rev, extra_args=extra_args) | |
@classmethod | |
def _is_local_repository(cls, repo): | |
# type: (str) -> bool | |
""" | |
posix absolute paths start with os.path.sep, | |
win32 ones start with drive (like c:\\folder) | |
""" | |
drive, tail = os.path.splitdrive(repo) | |
return repo.startswith(os.path.sep) or bool(drive) | |
def export(self, location, url): | |
# type: (str, HiddenText) -> None | |
""" | |
Export the repository at the url to the destination location | |
i.e. only download the files, without vcs informations | |
:param url: the repository URL starting with a vcs prefix. | |
""" | |
raise NotImplementedError | |
@classmethod | |
def get_netloc_and_auth(cls, netloc, scheme): | |
# type: (str, str) -> Tuple[str, Tuple[Optional[str], Optional[str]]] | |
""" | |
Parse the repository URL's netloc, and return the new netloc to use | |
along with auth information. | |
Args: | |
netloc: the original repository URL netloc. | |
scheme: the repository URL's scheme without the vcs prefix. | |
This is mainly for the Subversion class to override, so that auth | |
information can be provided via the --username and --password options | |
instead of through the URL. For other subclasses like Git without | |
such an option, auth information must stay in the URL. | |
Returns: (netloc, (username, password)). | |
""" | |
return netloc, (None, None) | |
@classmethod | |
def get_url_rev_and_auth(cls, url): | |
# type: (str) -> Tuple[str, Optional[str], AuthInfo] | |
""" | |
Parse the repository URL to use, and return the URL, revision, | |
and auth info to use. | |
Returns: (url, rev, (username, password)). | |
""" | |
scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) | |
if '+' not in scheme: | |
raise ValueError( | |
"Sorry, {!r} is a malformed VCS url. " | |
"The format is <vcs>+<protocol>://<url>, " | |
"e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) | |
) | |
# Remove the vcs prefix. | |
scheme = scheme.split('+', 1)[1] | |
netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) | |
rev = None | |
if '@' in path: | |
path, rev = path.rsplit('@', 1) | |
url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) | |
return url, rev, user_pass | |
@staticmethod | |
def make_rev_args(username, password): | |
# type: (Optional[str], Optional[HiddenText]) -> CommandArgs | |
""" | |
Return the RevOptions "extra arguments" to use in obtain(). | |
""" | |
return [] | |
def get_url_rev_options(self, url): | |
# type: (HiddenText) -> Tuple[HiddenText, RevOptions] | |
""" | |
Return the URL and RevOptions object to use in obtain() and in | |
some cases export(), as a tuple (url, rev_options). | |
""" | |
secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) | |
username, secret_password = user_pass | |
password = None # type: Optional[HiddenText] | |
if secret_password is not None: | |
password = hide_value(secret_password) | |
extra_args = self.make_rev_args(username, password) | |
rev_options = self.make_rev_options(rev, extra_args=extra_args) | |
return hide_url(secret_url), rev_options | |
@staticmethod | |
def normalize_url(url): | |
# type: (str) -> str | |
""" | |
Normalize a URL for comparison by unquoting it and removing any | |
trailing slash. | |
""" | |
return urllib_parse.unquote(url).rstrip('/') | |
@classmethod | |
def compare_urls(cls, url1, url2): | |
# type: (str, str) -> bool | |
""" | |
Compare two repo URLs for identity, ignoring incidental differences. | |
""" | |
return (cls.normalize_url(url1) == cls.normalize_url(url2)) | |
def fetch_new(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
""" | |
Fetch a revision from a repository, in the case that this is the | |
first fetch from the repository. | |
Args: | |
dest: the directory to fetch the repository to. | |
rev_options: a RevOptions object. | |
""" | |
raise NotImplementedError | |
def switch(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
""" | |
Switch the repo at ``dest`` to point to ``URL``. | |
Args: | |
rev_options: a RevOptions object. | |
""" | |
raise NotImplementedError | |
def update(self, dest, url, rev_options): | |
# type: (str, HiddenText, RevOptions) -> None | |
""" | |
Update an already-existing repo to the given ``rev_options``. | |
Args: | |
rev_options: a RevOptions object. | |
""" | |
raise NotImplementedError | |
@classmethod | |
def is_commit_id_equal(cls, dest, name): | |
# type: (str, Optional[str]) -> bool | |
""" | |
Return whether the id of the current commit equals the given name. | |
Args: | |
dest: the repository directory. | |
name: a string name. | |
""" | |
raise NotImplementedError | |
def obtain(self, dest, url): | |
# type: (str, HiddenText) -> None | |
""" | |
Install or update in editable mode the package represented by this | |
VersionControl object. | |
:param dest: the repository directory in which to install or update. | |
:param url: the repository URL starting with a vcs prefix. | |
""" | |
url, rev_options = self.get_url_rev_options(url) | |
if not os.path.exists(dest): | |
self.fetch_new(dest, url, rev_options) | |
return | |
rev_display = rev_options.to_display() | |
if self.is_repository_directory(dest): | |
existing_url = self.get_remote_url(dest) | |
if self.compare_urls(existing_url, url.secret): | |
logger.debug( | |
'%s in %s exists, and has correct URL (%s)', | |
self.repo_name.title(), | |
display_path(dest), | |
url, | |
) | |
if not self.is_commit_id_equal(dest, rev_options.rev): | |
logger.info( | |
'Updating %s %s%s', | |
display_path(dest), | |
self.repo_name, | |
rev_display, | |
) | |
self.update(dest, url, rev_options) | |
else: | |
logger.info('Skipping because already up-to-date.') | |
return | |
logger.warning( | |
'%s %s in %s exists with URL %s', | |
self.name, | |
self.repo_name, | |
display_path(dest), | |
existing_url, | |
) | |
prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', | |
('s', 'i', 'w', 'b')) | |
else: | |
logger.warning( | |
'Directory %s already exists, and is not a %s %s.', | |
dest, | |
self.name, | |
self.repo_name, | |
) | |
# https://github.com/python/mypy/issues/1174 | |
prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore | |
('i', 'w', 'b')) | |
logger.warning( | |
'The plan is to install the %s repository %s', | |
self.name, | |
url, | |
) | |
response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) | |
if response == 'a': | |
sys.exit(-1) | |
if response == 'w': | |
logger.warning('Deleting %s', display_path(dest)) | |
rmtree(dest) | |
self.fetch_new(dest, url, rev_options) | |
return | |
if response == 'b': | |
dest_dir = backup_dir(dest) | |
logger.warning( | |
'Backing up %s to %s', display_path(dest), dest_dir, | |
) | |
shutil.move(dest, dest_dir) | |
self.fetch_new(dest, url, rev_options) | |
return | |
# Do nothing if the response is "i". | |
if response == 's': | |
logger.info( | |
'Switching %s %s to %s%s', | |
self.repo_name, | |
display_path(dest), | |
url, | |
rev_display, | |
) | |
self.switch(dest, url, rev_options) | |
def unpack(self, location, url): | |
# type: (str, HiddenText) -> None | |
""" | |
Clean up current location and download the url repository | |
(and vcs infos) into location | |
:param url: the repository URL starting with a vcs prefix. | |
""" | |
if os.path.exists(location): | |
rmtree(location) | |
self.obtain(location, url=url) | |
@classmethod | |
def get_remote_url(cls, location): | |
# type: (str) -> str | |
""" | |
Return the url used at location | |
Raises RemoteNotFoundError if the repository does not have a remote | |
url configured. | |
""" | |
raise NotImplementedError | |
@classmethod | |
def get_revision(cls, location): | |
# type: (str) -> str | |
""" | |
Return the current commit id of the files at the given location. | |
""" | |
raise NotImplementedError | |
@classmethod | |
def run_command( | |
cls, | |
cmd, # type: Union[List[str], CommandArgs] | |
show_stdout=True, # type: bool | |
cwd=None, # type: Optional[str] | |
on_returncode='raise', # type: str | |
extra_ok_returncodes=None, # type: Optional[Iterable[int]] | |
command_desc=None, # type: Optional[str] | |
extra_environ=None, # type: Optional[Mapping[str, Any]] | |
spinner=None, # type: Optional[SpinnerInterface] | |
log_failed_cmd=True # type: bool | |
): | |
# type: (...) -> Text | |
""" | |
Run a VCS subcommand | |
This is simply a wrapper around call_subprocess that adds the VCS | |
command name, and checks that the VCS is available | |
""" | |
cmd = make_command(cls.name, *cmd) | |
try: | |
return call_subprocess(cmd, show_stdout, cwd, | |
on_returncode=on_returncode, | |
extra_ok_returncodes=extra_ok_returncodes, | |
command_desc=command_desc, | |
extra_environ=extra_environ, | |
unset_environ=cls.unset_environ, | |
spinner=spinner, | |
log_failed_cmd=log_failed_cmd) | |
except OSError as e: | |
# errno.ENOENT = no such file or directory | |
# In other words, the VCS executable isn't available | |
if e.errno == errno.ENOENT: | |
raise BadCommand( | |
'Cannot find command %r - do you have ' | |
'%r installed and in your ' | |
'PATH?' % (cls.name, cls.name)) | |
else: | |
raise # re-raise exception if a different error occurred | |
@classmethod | |
def is_repository_directory(cls, path): | |
# type: (str) -> bool | |
""" | |
Return whether a directory path is a repository directory. | |
""" | |
logger.debug('Checking in %s for %s (%s)...', | |
path, cls.dirname, cls.name) | |
return os.path.exists(os.path.join(path, cls.dirname)) | |
@classmethod | |
def controls_location(cls, location): | |
# type: (str) -> bool | |
""" | |
Check if a location is controlled by the vcs. | |
It is meant to be overridden to implement smarter detection | |
mechanisms for specific vcs. | |
This can do more than is_repository_directory() alone. For example, | |
the Git override checks that Git is actually available. | |
""" | |
return cls.is_repository_directory(location) |
"""Orchestrator for building wheels from InstallRequirements. | |
""" | |
# The following comment should be removed at some point in the future. | |
# mypy: strict-optional=False | |
import logging | |
import os.path | |
import re | |
import shutil | |
from pip._internal.models.link import Link | |
from pip._internal.operations.build.wheel import build_wheel_pep517 | |
from pip._internal.operations.build.wheel_legacy import build_wheel_legacy | |
from pip._internal.utils.logging import indent_log | |
from pip._internal.utils.misc import ensure_dir, hash_file, is_wheel_installed | |
from pip._internal.utils.setuptools_build import make_setuptools_clean_args | |
from pip._internal.utils.subprocess import call_subprocess | |
from pip._internal.utils.temp_dir import TempDirectory | |
from pip._internal.utils.typing import MYPY_CHECK_RUNNING | |
from pip._internal.utils.urls import path_to_url | |
from pip._internal.vcs import vcs | |
if MYPY_CHECK_RUNNING: | |
from typing import ( | |
Any, Callable, Iterable, List, Optional, Pattern, Tuple, | |
) | |
from pip._internal.cache import WheelCache | |
from pip._internal.req.req_install import InstallRequirement | |
BinaryAllowedPredicate = Callable[[InstallRequirement], bool] | |
BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]] | |
logger = logging.getLogger(__name__) | |
def _contains_egg_info( | |
s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): | |
# type: (str, Pattern[str]) -> bool | |
"""Determine whether the string looks like an egg_info. | |
:param s: The string to parse. E.g. foo-2.1 | |
""" | |
return bool(_egg_info_re.search(s)) | |
def _should_build( | |
req, # type: InstallRequirement | |
need_wheel, # type: bool | |
check_binary_allowed, # type: BinaryAllowedPredicate | |
): | |
# type: (...) -> bool | |
"""Return whether an InstallRequirement should be built into a wheel.""" | |
if req.constraint: | |
# never build requirements that are merely constraints | |
return False | |
if req.is_wheel: | |
if need_wheel: | |
logger.info( | |
'Skipping %s, due to already being wheel.', req.name, | |
) | |
return False | |
if need_wheel: | |
# i.e. pip wheel, not pip install | |
return True | |
# From this point, this concerns the pip install command only | |
# (need_wheel=False). | |
if not req.use_pep517 and not is_wheel_installed(): | |
# we don't build legacy requirements if wheel is not installed | |
return False | |
if req.editable or not req.source_dir: | |
return False | |
if not check_binary_allowed(req): | |
logger.info( | |
"Skipping wheel build for %s, due to binaries " | |
"being disabled for it.", req.name, | |
) | |
return False | |
return True | |
def should_build_for_wheel_command( | |
req, # type: InstallRequirement | |
): | |
# type: (...) -> bool | |
return _should_build( | |
req, need_wheel=True, check_binary_allowed=_always_true | |
) | |
def should_build_for_install_command( | |
req, # type: InstallRequirement | |
check_binary_allowed, # type: BinaryAllowedPredicate | |
): | |
# type: (...) -> bool | |
return _should_build( | |
req, need_wheel=False, check_binary_allowed=check_binary_allowed | |
) | |
def _should_cache( | |
req, # type: InstallRequirement | |
): | |
# type: (...) -> Optional[bool] | |
""" | |
Return whether a built InstallRequirement can be stored in the persistent | |
wheel cache, assuming the wheel cache is available, and _should_build() | |
has determined a wheel needs to be built. | |
""" | |
if not should_build_for_install_command( | |
req, check_binary_allowed=_always_true | |
): | |
# never cache if pip install would not have built | |
# (editable mode, etc) | |
return False | |
if req.link and req.link.is_vcs: | |
# VCS checkout. Do not cache | |
# unless it points to an immutable commit hash. | |
assert not req.editable | |
assert req.source_dir | |
vcs_backend = vcs.get_backend_for_scheme(req.link.scheme) | |
assert vcs_backend | |
if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir): | |
return True | |
return False | |
base, ext = req.link.splitext() | |
if _contains_egg_info(base): | |
return True | |
# Otherwise, do not cache. | |
return False | |
def _get_cache_dir( | |
req, # type: InstallRequirement | |
wheel_cache, # type: WheelCache | |
): | |
# type: (...) -> str | |
"""Return the persistent or temporary cache directory where the built | |
wheel need to be stored. | |
""" | |
cache_available = bool(wheel_cache.cache_dir) | |
if cache_available and _should_cache(req): | |
cache_dir = wheel_cache.get_path_for_link(req.link) | |
else: | |
cache_dir = wheel_cache.get_ephem_path_for_link(req.link) | |
return cache_dir | |
def _always_true(_): | |
# type: (Any) -> bool | |
return True | |
def _build_one( | |
req, # type: InstallRequirement | |
output_dir, # type: str | |
build_options, # type: List[str] | |
global_options, # type: List[str] | |
): | |
# type: (...) -> Optional[str] | |
"""Build one wheel. | |
:return: The filename of the built wheel, or None if the build failed. | |
""" | |
try: | |
ensure_dir(output_dir) | |
except OSError as e: | |
logger.warning( | |
"Building wheel for %s failed: %s", | |
req.name, e, | |
) | |
return None | |
# Install build deps into temporary directory (PEP 518) | |
with req.build_env: | |
return _build_one_inside_env( | |
req, output_dir, build_options, global_options | |
) | |
def _build_one_inside_env( | |
req, # type: InstallRequirement | |
output_dir, # type: str | |
build_options, # type: List[str] | |
global_options, # type: List[str] | |
): | |
# type: (...) -> Optional[str] | |
with TempDirectory(kind="wheel") as temp_dir: | |
if req.use_pep517: | |
wheel_path = build_wheel_pep517( | |
name=req.name, | |
backend=req.pep517_backend, | |
metadata_directory=req.metadata_directory, | |
build_options=build_options, | |
tempd=temp_dir.path, | |
) | |
else: | |
wheel_path = build_wheel_legacy( | |
name=req.name, | |
setup_py_path=req.setup_py_path, | |
source_dir=req.unpacked_source_directory, | |
global_options=global_options, | |
build_options=build_options, | |
tempd=temp_dir.path, | |
) | |
if wheel_path is not None: | |
wheel_name = os.path.basename(wheel_path) | |
dest_path = os.path.join(output_dir, wheel_name) | |
try: | |
wheel_hash, length = hash_file(wheel_path) | |
shutil.move(wheel_path, dest_path) | |
logger.info('Created wheel for %s: ' | |
'filename=%s size=%d sha256=%s', | |
req.name, wheel_name, length, | |
wheel_hash.hexdigest()) | |
logger.info('Stored in directory: %s', output_dir) | |
return dest_path | |
except Exception as e: | |
logger.warning( | |
"Building wheel for %s failed: %s", | |
req.name, e, | |
) | |
# Ignore return, we can't do anything else useful. | |
if not req.use_pep517: | |
_clean_one_legacy(req, global_options) | |
return None | |
def _clean_one_legacy(req, global_options): | |
# type: (InstallRequirement, List[str]) -> bool | |
clean_args = make_setuptools_clean_args( | |
req.setup_py_path, | |
global_options=global_options, | |
) | |
logger.info('Running setup.py clean for %s', req.name) | |
try: | |
call_subprocess(clean_args, cwd=req.source_dir) | |
return True | |
except Exception: | |
logger.error('Failed cleaning build dir for %s', req.name) | |
return False | |
def build( | |
requirements, # type: Iterable[InstallRequirement] | |
wheel_cache, # type: WheelCache | |
build_options, # type: List[str] | |
global_options, # type: List[str] | |
): | |
# type: (...) -> BuildResult | |
"""Build wheels. | |
:return: The list of InstallRequirement that succeeded to build and | |
the list of InstallRequirement that failed to build. | |
""" | |
if not requirements: | |
return [], [] | |
# Build the wheels. | |
logger.info( | |
'Building wheels for collected packages: %s', | |
', '.join(req.name for req in requirements), | |
) | |
with indent_log(): | |
build_successes, build_failures = [], [] | |
for req in requirements: | |
cache_dir = _get_cache_dir(req, wheel_cache) | |
wheel_file = _build_one( | |
req, cache_dir, build_options, global_options | |
) | |
if wheel_file: | |
# Update the link for this. | |
req.link = Link(path_to_url(wheel_file)) | |
req.local_file_path = req.link.file_path | |
assert req.link.is_wheel | |
build_successes.append(req) | |
else: | |
build_failures.append(req) | |
# notify success/failure | |
if build_successes: | |
logger.info( | |
'Successfully built %s', | |
' '.join([req.name for req in build_successes]), | |
) | |
if build_failures: | |
logger.info( | |
'Failed to build %s', | |
' '.join([req.name for req in build_failures]), | |
) | |
# Return a list of requirements that failed to build | |
return build_successes, build_failures |
""" | |
pip._vendor is for vendoring dependencies of pip to prevent needing pip to | |
depend on something external. | |
Files inside of pip._vendor should be considered immutable and should only be | |
updated to versions from upstream. | |
""" | |
from __future__ import absolute_import | |
import glob | |
import os.path | |
import sys | |
# Downstream redistributors which have debundled our dependencies should also | |
# patch this value to be true. This will trigger the additional patching | |
# to cause things like "six" to be available as pip. | |
DEBUNDLED = True | |
# By default, look in this directory for a bunch of .whl files which we will | |
# add to the beginning of sys.path before attempting to import anything. This | |
# is done to support downstream re-distributors like Debian and Fedora who | |
# wish to create their own Wheels for our dependencies to aid in debundling. | |
prefix = sys.prefix | |
if sys.prefix.startswith('/usr/lib/pypy'): | |
prefix = '/usr' | |
WHEEL_DIR = os.path.abspath(os.path.join(prefix, 'share', 'python-wheels')) | |
# Define a small helper function to alias our vendored modules to the real ones | |
# if the vendored ones do not exist. This idea of this was taken from | |
# https://github.com/kennethreitz/requests/pull/2567. | |
def vendored(modulename): | |
vendored_name = "{0}.{1}".format(__name__, modulename) | |
try: | |
__import__(modulename, globals(), locals(), level=0) | |
except ImportError: | |
# We can just silently allow import failures to pass here. If we | |
# got to this point it means that ``import pip._vendor.whatever`` | |
# failed and so did ``import whatever``. Since we're importing this | |
# upfront in an attempt to alias imports, not erroring here will | |
# just mean we get a regular import error whenever pip *actually* | |
# tries to import one of these modules to use it, which actually | |
# gives us a better error message than we would have otherwise | |
# gotten. | |
pass | |
else: | |
sys.modules[vendored_name] = sys.modules[modulename] | |
base, head = vendored_name.rsplit(".", 1) | |
setattr(sys.modules[base], head, sys.modules[modulename]) | |
# If we're operating in a debundled setup, then we want to go ahead and trigger | |
# the aliasing of our vendored libraries as well as looking for wheels to add | |
# to our sys.path. This will cause all of this code to be a no-op typically | |
# however downstream redistributors can enable it in a consistent way across | |
# all platforms. | |
if DEBUNDLED: | |
# Actually look inside of WHEEL_DIR to find .whl files and add them to the | |
# front of our sys.path. | |
sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path | |
# Actually alias all of our vendored dependencies. | |
vendored("appdirs") | |
vendored("cachecontrol") | |
vendored("colorama") | |
vendored("contextlib2") | |
vendored("distlib") | |
vendored("distro") | |
vendored("html5lib") | |
vendored("six") | |
vendored("six.moves") | |
vendored("six.moves.urllib") | |
vendored("six.moves.urllib.parse") | |
vendored("packaging") | |
vendored("packaging.version") | |
vendored("packaging.specifiers") | |
vendored("pep517") | |
vendored("pkg_resources") | |
vendored("progress") | |
vendored("pytoml") | |
vendored("retrying") | |
vendored("requests") | |
vendored("requests.exceptions") | |
vendored("requests.packages") | |
vendored("requests.packages.urllib3") | |
vendored("requests.packages.urllib3._collections") | |
vendored("requests.packages.urllib3.connection") | |
vendored("requests.packages.urllib3.connectionpool") | |
vendored("requests.packages.urllib3.contrib") | |
vendored("requests.packages.urllib3.contrib.ntlmpool") | |
vendored("requests.packages.urllib3.contrib.pyopenssl") | |
vendored("requests.packages.urllib3.exceptions") | |
vendored("requests.packages.urllib3.fields") | |
vendored("requests.packages.urllib3.filepost") | |
vendored("requests.packages.urllib3.packages") | |
try: | |
vendored("requests.packages.urllib3.packages.ordered_dict") | |
vendored("requests.packages.urllib3.packages.six") | |
except ImportError: | |
# Debian already unbundles these from requests. | |
pass | |
vendored("requests.packages.urllib3.packages.ssl_match_hostname") | |
vendored("requests.packages.urllib3.packages.ssl_match_hostname." | |
"_implementation") | |
vendored("requests.packages.urllib3.poolmanager") | |
vendored("requests.packages.urllib3.request") | |
vendored("requests.packages.urllib3.response") | |
vendored("requests.packages.urllib3.util") | |
vendored("requests.packages.urllib3.util.connection") | |
vendored("requests.packages.urllib3.util.request") | |
vendored("requests.packages.urllib3.util.response") | |
vendored("requests.packages.urllib3.util.retry") | |
vendored("requests.packages.urllib3.util.ssl_") | |
vendored("requests.packages.urllib3.util.timeout") | |
vendored("requests.packages.urllib3.util.url") | |
vendored("urllib3") |
pip |
Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) | |
Permission is hereby granted, free of charge, to any person obtaining | |
a copy of this software and associated documentation files (the | |
"Software"), to deal in the Software without restriction, including | |
without limitation the rights to use, copy, modify, merge, publish, | |
distribute, sublicense, and/or sell copies of the Software, and to | |
permit persons to whom the Software is furnished to do so, subject to | |
the following conditions: | |
The above copyright notice and this permission notice shall be | |
included in all copies or substantial portions of the Software. | |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE | |
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION | |
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
Metadata-Version: 2.1 | |
Name: pkg_resources | |
Version: 0.0.0 | |
Summary: UNKNOWN | |
Home-page: UNKNOWN | |
Author: UNKNOWN | |
Author-email: UNKNOWN | |
License: UNKNOWN | |
Platform: UNKNOWN | |
UNKNOWN | |
pkg_resources-0.0.0.dist-info/AUTHORS.txt,sha256=RtqU9KfonVGhI48DAA4-yTOBUhBtQTjFhaDzHoyh7uU,21518 | |
pkg_resources-0.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 | |
pkg_resources-0.0.0.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090 | |
pkg_resources-0.0.0.dist-info/METADATA,sha256=V9_WPOtD1FnuKrTGv6Ique7kAOn2lasvT8W0_iMCCCk,177 | |
pkg_resources-0.0.0.dist-info/RECORD,, | |
pkg_resources-0.0.0.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 | |
pkg_resources/__init__.py,sha256=0IssxXPnaDKpYZRra8Ime0JG4hwosQljItGD0bnIkGk,108349 | |
pkg_resources/__pycache__/__init__.cpython-38.pyc,, | |
pkg_resources/__pycache__/py31compat.cpython-38.pyc,, | |
pkg_resources/_vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 | |
pkg_resources/_vendor/__pycache__/__init__.cpython-38.pyc,, | |
pkg_resources/_vendor/__pycache__/appdirs.cpython-38.pyc,, | |
pkg_resources/_vendor/__pycache__/pyparsing.cpython-38.pyc,, | |
pkg_resources/_vendor/__pycache__/six.cpython-38.pyc,, | |
pkg_resources/_vendor/appdirs.py,sha256=MievUEuv3l_mQISH5SF0shDk_BNhHHzYiAPrT3ITN4I,24701 | |
pkg_resources/_vendor/packaging/__about__.py,sha256=zkcCPTN_6TcLW0Nrlg0176-R1QQ_WVPTm8sz1R4-HjM,720 | |
pkg_resources/_vendor/packaging/__init__.py,sha256=_vNac5TrzwsrzbOFIbF-5cHqc_Y2aPT2D7zrIR06BOo,513 | |
pkg_resources/_vendor/packaging/__pycache__/__about__.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/__init__.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/_compat.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/_structures.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/markers.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/requirements.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/specifiers.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/utils.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/__pycache__/version.cpython-38.pyc,, | |
pkg_resources/_vendor/packaging/_compat.py,sha256=Vi_A0rAQeHbU-a9X0tt1yQm9RqkgQbDSxzRw8WlU9kA,860 | |
pkg_resources/_vendor/packaging/_structures.py,sha256=RImECJ4c_wTlaTYYwZYLHEiebDMaAJmK1oPARhw1T5o,1416 | |
pkg_resources/_vendor/packaging/markers.py,sha256=uEcBBtGvzqltgnArqb9c4RrcInXezDLos14zbBHhWJo,8248 | |
pkg_resources/_vendor/packaging/requirements.py,sha256=SikL2UynbsT0qtY9ltqngndha_sfo0w6XGFhAhoSoaQ,4355 | |
pkg_resources/_vendor/packaging/specifiers.py,sha256=SAMRerzO3fK2IkFZCaZkuwZaL_EGqHNOz4pni4vhnN0,28025 | |
pkg_resources/_vendor/packaging/utils.py,sha256=3m6WvPm6NNxE8rkTGmn0r75B_GZSGg7ikafxHsBN1WA,421 | |
pkg_resources/_vendor/packaging/version.py,sha256=OwGnxYfr2ghNzYx59qWIBkrK3SnB6n-Zfd1XaLpnnM0,11556 | |
pkg_resources/_vendor/pyparsing.py,sha256=tmrp-lu-qO1i75ZzIN5A12nKRRD1Cm4Vpk-5LR9rims,232055 | |
pkg_resources/_vendor/six.py,sha256=A6hdJZVjI3t_geebZ9BzUvwRrIXo0lfwzQlM2LcKyas,30098 | |
pkg_resources/extern/__init__.py,sha256=cHiEfHuLmm6rs5Ve_ztBfMI7Lr31vss-D4wkqF5xzlI,2498 | |
pkg_resources/extern/__pycache__/__init__.cpython-38.pyc,, | |
pkg_resources/py31compat.py,sha256=-WQ0e4c3RG_acdhwC3gLiXhP_lg4G5q7XYkZkQg0gxU,558 |
Wheel-Version: 1.0 | |
Generator: bdist_wheel (0.34.2) | |
Root-Is-Purelib: true | |
Tag: py2-none-any | |
Tag: py3-none-any | |
# coding: utf-8 | |
""" | |
Package resource API | |
-------------------- | |
A resource is a logical file contained within a package, or a logical | |
subdirectory thereof. The package resource API expects resource names | |
to have their path parts separated with ``/``, *not* whatever the local | |
path separator is. Do not use os.path operations to manipulate resource | |
names being passed into the API. | |
The package resource API is designed to work with normal filesystem packages, | |
.egg files, and unpacked .egg files. It can also work in a limited way with | |
.zip files and with custom PEP 302 loaders that support the ``get_data()`` | |
method. | |
""" | |
from __future__ import absolute_import | |
import sys | |
import os | |
import io | |
import time | |
import re | |
import types | |
import zipfile | |
import zipimport | |
import warnings | |
import stat | |
import functools | |
import pkgutil | |
import operator | |
import platform | |
import collections | |
import plistlib | |
import email.parser | |
import errno | |
import tempfile | |
import textwrap | |
import itertools | |
import inspect | |
import ntpath | |
import posixpath | |
from pkgutil import get_importer | |
try: | |
import _imp | |
except ImportError: | |
# Python 3.2 compatibility | |
import imp as _imp | |
try: | |
FileExistsError | |
except NameError: | |
FileExistsError = OSError | |
from pkg_resources.extern import six | |
from pkg_resources.extern.six.moves import urllib, map, filter | |
# capture these to bypass sandboxing | |
from os import utime | |
try: | |
from os import mkdir, rename, unlink | |
WRITE_SUPPORT = True | |
except ImportError: | |
# no write support, probably under GAE | |
WRITE_SUPPORT = False | |
from os import open as os_open | |
from os.path import isdir, split | |
try: | |
import importlib.machinery as importlib_machinery | |
# access attribute to force import under delayed import mechanisms. | |
importlib_machinery.__name__ | |
except ImportError: | |
importlib_machinery = None | |
from . import py31compat | |
from pkg_resources.extern import appdirs | |
from pkg_resources.extern import packaging | |
__import__('pkg_resources.extern.packaging.version') | |
__import__('pkg_resources.extern.packaging.specifiers') | |
__import__('pkg_resources.extern.packaging.requirements') | |
__import__('pkg_resources.extern.packaging.markers') | |
__metaclass__ = type | |
if (3, 0) < sys.version_info < (3, 5): | |
raise RuntimeError("Python 3.5 or later is required") | |
if six.PY2: | |
# Those builtin exceptions are only defined in Python 3 | |
PermissionError = None | |
NotADirectoryError = None | |
# declare some globals that will be defined later to | |
# satisfy the linters. | |
require = None | |
working_set = None | |
add_activation_listener = None | |
resources_stream = None | |
cleanup_resources = None | |
resource_dir = None | |
resource_stream = None | |
set_extraction_path = None | |
resource_isdir = None | |
resource_string = None | |
iter_entry_points = None | |
resource_listdir = None | |
resource_filename = None | |
resource_exists = None | |
_distribution_finders = None | |
_namespace_handlers = None | |
_namespace_packages = None | |
class PEP440Warning(RuntimeWarning): | |
""" | |
Used when there is an issue with a version or specifier not complying with | |
PEP 440. | |
""" | |
def parse_version(v): | |
try: | |
return packaging.version.Version(v) | |
except packaging.version.InvalidVersion: | |
return packaging.version.LegacyVersion(v) | |
_state_vars = {} | |
def _declare_state(vartype, **kw): | |
globals().update(kw) | |
_state_vars.update(dict.fromkeys(kw, vartype)) | |
def __getstate__(): | |
state = {} | |
g = globals() | |
for k, v in _state_vars.items(): | |
state[k] = g['_sget_' + v](g[k]) | |
return state | |
def __setstate__(state): | |
g = globals() | |
for k, v in state.items(): | |
g['_sset_' + _state_vars[k]](k, g[k], v) | |
return state | |
def _sget_dict(val): | |
return val.copy() | |
def _sset_dict(key, ob, state): | |
ob.clear() | |
ob.update(state) | |
def _sget_object(val): | |
return val.__getstate__() | |
def _sset_object(key, ob, state): | |
ob.__setstate__(state) | |
_sget_none = _sset_none = lambda *args: None | |
def get_supported_platform(): | |
"""Return this platform's maximum compatible version. | |
distutils.util.get_platform() normally reports the minimum version | |
of Mac OS X that would be required to *use* extensions produced by | |
distutils. But what we want when checking compatibility is to know the | |
version of Mac OS X that we are *running*. To allow usage of packages that | |
explicitly require a newer version of Mac OS X, we must also know the | |
current version of the OS. | |
If this condition occurs for any other platform with a version in its | |
platform strings, this function should be extended accordingly. | |
""" | |
plat = get_build_platform() | |
m = macosVersionString.match(plat) | |
if m is not None and sys.platform == "darwin": | |
try: | |
plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3)) | |
except ValueError: | |
# not Mac OS X | |
pass | |
return plat | |
__all__ = [ | |
# Basic resource access and distribution/entry point discovery | |
'require', 'run_script', 'get_provider', 'get_distribution', | |
'load_entry_point', 'get_entry_map', 'get_entry_info', | |
'iter_entry_points', | |
'resource_string', 'resource_stream', 'resource_filename', | |
'resource_listdir', 'resource_exists', 'resource_isdir', | |
# Environmental control | |
'declare_namespace', 'working_set', 'add_activation_listener', | |
'find_distributions', 'set_extraction_path', 'cleanup_resources', | |
'get_default_cache', | |
# Primary implementation classes | |
'Environment', 'WorkingSet', 'ResourceManager', | |
'Distribution', 'Requirement', 'EntryPoint', | |
# Exceptions | |
'ResolutionError', 'VersionConflict', 'DistributionNotFound', | |
'UnknownExtra', 'ExtractionError', | |
# Warnings | |
'PEP440Warning', | |
# Parsing functions and string utilities | |
'parse_requirements', 'parse_version', 'safe_name', 'safe_version', | |
'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections', | |
'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker', | |
# filesystem utilities | |
'ensure_directory', 'normalize_path', | |
# Distribution "precedence" constants | |
'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST', | |
# "Provider" interfaces, implementations, and registration/lookup APIs | |
'IMetadataProvider', 'IResourceProvider', 'FileMetadata', | |
'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider', | |
'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider', | |
'register_finder', 'register_namespace_handler', 'register_loader_type', | |
'fixup_namespace_packages', 'get_importer', | |
# Warnings | |
'PkgResourcesDeprecationWarning', | |
# Deprecated/backward compatibility only | |
'run_main', 'AvailableDistributions', | |
] | |
class ResolutionError(Exception): | |
"""Abstract base for dependency resolution errors""" | |
def __repr__(self): | |
return self.__class__.__name__ + repr(self.args) | |
class VersionConflict(ResolutionError): | |
""" | |
An already-installed version conflicts with the requested version. | |
Should be initialized with the installed Distribution and the requested | |
Requirement. | |
""" | |
_template = "{self.dist} is installed but {self.req} is required" | |
@property | |
def dist(self): | |
return self.args[0] | |
@property | |
def req(self): | |
return self.args[1] | |
def report(self): | |
return self._template.format(**locals()) | |
def with_context(self, required_by): | |
""" | |
If required_by is non-empty, return a version of self that is a | |
ContextualVersionConflict. | |
""" | |
if not required_by: | |
return self | |
args = self.args + (required_by,) | |
return ContextualVersionConflict(*args) | |
class ContextualVersionConflict(VersionConflict): | |
""" | |
A VersionConflict that accepts a third parameter, the set of the | |
requirements that required the installed Distribution. | |
""" | |
_template = VersionConflict._template + ' by {self.required_by}' | |
@property | |
def required_by(self): | |
return self.args[2] | |
class DistributionNotFound(ResolutionError): | |
"""A requested distribution was not found""" | |
_template = ("The '{self.req}' distribution was not found " | |
"and is required by {self.requirers_str}") | |
@property | |
def req(self): | |
return self.args[0] | |
@property | |
def requirers(self): | |
return self.args[1] | |
@property | |
def requirers_str(self): | |
if not self.requirers: | |
return 'the application' | |
return ', '.join(self.requirers) | |
def report(self): | |
return self._template.format(**locals()) | |
def __str__(self): | |
return self.report() | |
class UnknownExtra(ResolutionError): | |
"""Distribution doesn't have an "extra feature" of the given name""" | |
_provider_factories = {} | |
PY_MAJOR = '{}.{}'.format(*sys.version_info) | |
EGG_DIST = 3 | |
BINARY_DIST = 2 | |
SOURCE_DIST = 1 | |
CHECKOUT_DIST = 0 | |
DEVELOP_DIST = -1 | |
def register_loader_type(loader_type, provider_factory): | |
"""Register `provider_factory` to make providers for `loader_type` | |
`loader_type` is the type or class of a PEP 302 ``module.__loader__``, | |
and `provider_factory` is a function that, passed a *module* object, | |
returns an ``IResourceProvider`` for that module. | |
""" | |
_provider_factories[loader_type] = provider_factory | |
def get_provider(moduleOrReq): | |
"""Return an IResourceProvider for the named module or requirement""" | |
if isinstance(moduleOrReq, Requirement): | |
return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0] | |
try: | |
module = sys.modules[moduleOrReq] | |
except KeyError: | |
__import__(moduleOrReq) | |
module = sys.modules[moduleOrReq] | |
loader = getattr(module, '__loader__', None) | |
return _find_adapter(_provider_factories, loader)(module) | |
def _macosx_vers(_cache=[]): | |
if not _cache: | |
version = platform.mac_ver()[0] | |
# fallback for MacPorts | |
if version == '': | |
plist = '/System/Library/CoreServices/SystemVersion.plist' | |
if os.path.exists(plist): | |
if hasattr(plistlib, 'readPlist'): | |
plist_content = plistlib.readPlist(plist) | |
if 'ProductVersion' in plist_content: | |
version = plist_content['ProductVersion'] | |
_cache.append(version.split('.')) | |
return _cache[0] | |
def _macosx_arch(machine): | |
return {'PowerPC': 'ppc', 'Power_Macintosh': 'ppc'}.get(machine, machine) | |
def get_build_platform(): | |
"""Return this platform's string for platform-specific distributions | |
XXX Currently this is the same as ``distutils.util.get_platform()``, but it | |
needs some hacks for Linux and Mac OS X. | |
""" | |
from sysconfig import get_platform | |
plat = get_platform() | |
if sys.platform == "darwin" and not plat.startswith('macosx-'): | |
try: | |
version = _macosx_vers() | |
machine = os.uname()[4].replace(" ", "_") | |
return "macosx-%d.%d-%s" % ( | |
int(version[0]), int(version[1]), | |
_macosx_arch(machine), | |
) | |
except ValueError: | |
# if someone is running a non-Mac darwin system, this will fall | |
# through to the default implementation | |
pass | |
return plat | |
macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") | |
darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") | |
# XXX backward compat | |
get_platform = get_build_platform | |
def compatible_platforms(provided, required): | |
"""Can code for the `provided` platform run on the `required` platform? | |
Returns true if either platform is ``None``, or the platforms are equal. | |
XXX Needs compatibility checks for Linux and other unixy OSes. | |
""" | |
if provided is None or required is None or provided == required: | |
# easy case | |
return True | |
# Mac OS X special cases | |
reqMac = macosVersionString.match(required) | |
if reqMac: | |
provMac = macosVersionString.match(provided) | |
# is this a Mac package? | |
if not provMac: | |
# this is backwards compatibility for packages built before | |
# setuptools 0.6. All packages built after this point will | |
# use the new macosx designation. | |
provDarwin = darwinVersionString.match(provided) | |
if provDarwin: | |
dversion = int(provDarwin.group(1)) | |
macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2)) | |
if dversion == 7 and macosversion >= "10.3" or \ | |
dversion == 8 and macosversion >= "10.4": | |
return True | |
# egg isn't macosx or legacy darwin | |
return False | |
# are they the same major version and machine type? | |
if provMac.group(1) != reqMac.group(1) or \ | |
provMac.group(3) != reqMac.group(3): | |
return False | |
# is the required OS major update >= the provided one? | |
if int(provMac.group(2)) > int(reqMac.group(2)): | |
return False | |
return True | |
# XXX Linux and other platforms' special cases should go here | |
return False | |
def run_script(dist_spec, script_name): | |
"""Locate distribution `dist_spec` and run its `script_name` script""" | |
ns = sys._getframe(1).f_globals | |
name = ns['__name__'] | |
ns.clear() | |
ns['__name__'] = name | |
require(dist_spec)[0].run_script(script_name, ns) | |
# backward compatibility | |
run_main = run_script | |
def get_distribution(dist): | |
"""Return a current distribution object for a Requirement or string""" | |
if isinstance(dist, six.string_types): | |
dist = Requirement.parse(dist) | |
if isinstance(dist, Requirement): | |
dist = get_provider(dist) | |
if not isinstance(dist, Distribution): | |
raise TypeError("Expected string, Requirement, or Distribution", dist) | |
return dist | |
def load_entry_point(dist, group, name): | |
"""Return `name` entry point of `group` for `dist` or raise ImportError""" | |
return get_distribution(dist).load_entry_point(group, name) | |
def get_entry_map(dist, group=None): | |
"""Return the entry point map for `group`, or the full entry map""" | |
return get_distribution(dist).get_entry_map(group) | |
def get_entry_info(dist, group, name): | |
"""Return the EntryPoint object for `group`+`name`, or ``None``""" | |
return get_distribution(dist).get_entry_info(group, name) | |
class IMetadataProvider: | |
def has_metadata(name): | |
"""Does the package's distribution contain the named metadata?""" | |
def get_metadata(name): | |
"""The named metadata resource as a string""" | |
def get_metadata_lines(name): | |
"""Yield named metadata resource as list of non-blank non-comment lines | |
Leading and trailing whitespace is stripped from each line, and lines | |
with ``#`` as the first non-blank character are omitted.""" | |
def metadata_isdir(name): | |
"""Is the named metadata a directory? (like ``os.path.isdir()``)""" | |
def metadata_listdir(name): | |
"""List of metadata names in the directory (like ``os.listdir()``)""" | |
def run_script(script_name, namespace): | |
"""Execute the named script in the supplied namespace dictionary""" | |
class IResourceProvider(IMetadataProvider): | |
"""An object that provides access to package resources""" | |
def get_resource_filename(manager, resource_name): | |
"""Return a true filesystem path for `resource_name` | |
`manager` must be an ``IResourceManager``""" | |
def get_resource_stream(manager, resource_name): | |
"""Return a readable file-like object for `resource_name` | |
`manager` must be an ``IResourceManager``""" | |
def get_resource_string(manager, resource_name): | |
"""Return a string containing the contents of `resource_name` | |
`manager` must be an ``IResourceManager``""" | |
def has_resource(resource_name): | |
"""Does the package contain the named resource?""" | |
def resource_isdir(resource_name): | |
"""Is the named resource a directory? (like ``os.path.isdir()``)""" | |
def resource_listdir(resource_name): | |
"""List of resource names in the directory (like ``os.listdir()``)""" | |
class WorkingSet: | |
"""A collection of active distributions on sys.path (or a similar list)""" | |
def __init__(self, entries=None): | |
"""Create working set from list of path entries (default=sys.path)""" | |
self.entries = [] | |
self.entry_keys = {} | |
self.by_key = {} | |
self.callbacks = [] | |
if entries is None: | |
entries = sys.path | |
for entry in entries: | |
self.add_entry(entry) | |
@classmethod | |
def _build_master(cls): | |
""" | |
Prepare the master working set. | |
""" | |
ws = cls() | |
try: | |
from __main__ import __requires__ | |
except ImportError: | |
# The main program does not list any requirements | |
return ws | |
# ensure the requirements are met | |
try: | |
ws.require(__requires__) | |
except VersionConflict: | |
return cls._build_from_requirements(__requires__) | |
return ws | |
@classmethod | |
def _build_from_requirements(cls, req_spec): | |
""" | |
Build a working set from a requirement spec. Rewrites sys.path. | |
""" | |
# try it without defaults already on sys.path | |
# by starting with an empty path | |
ws = cls([]) | |
reqs = parse_requirements(req_spec) | |
dists = ws.resolve(reqs, Environment()) | |
for dist in dists: | |
ws.add(dist) | |
# add any missing entries from sys.path | |
for entry in sys.path: | |
if entry not in ws.entries: | |
ws.add_entry(entry) | |
# then copy back to sys.path | |
sys.path[:] = ws.entries | |
return ws | |
def add_entry(self, entry): | |
"""Add a path item to ``.entries``, finding any distributions on it | |
``find_distributions(entry, True)`` is used to find distributions | |
corresponding to the path entry, and they are added. `entry` is | |
always appended to ``.entries``, even if it is already present. | |
(This is because ``sys.path`` can contain the same value more than | |
once, and the ``.entries`` of the ``sys.path`` WorkingSet should always | |
equal ``sys.path``.) | |
""" | |
self.entry_keys.setdefault(entry, []) | |
self.entries.append(entry) | |
for dist in find_distributions(entry, True): | |
self.add(dist, entry, False) | |
def __contains__(self, dist): | |
"""True if `dist` is the active distribution for its project""" | |
return self.by_key.get(dist.key) == dist | |
def find(self, req): | |
"""Find a distribution matching requirement `req` | |
If there is an active distribution for the requested project, this | |
returns it as long as it meets the version requirement specified by | |
`req`. But, if there is an active distribution for the project and it | |
does *not* meet the `req` requirement, ``VersionConflict`` is raised. | |
If there is no active distribution for the requested project, ``None`` | |
is returned. | |
""" | |
dist = self.by_key.get(req.key) | |
if dist is not None and dist not in req: | |
# XXX add more info | |
raise VersionConflict(dist, req) | |
return dist | |
def iter_entry_points(self, group, name=None): | |
"""Yield entry point objects from `group` matching `name` | |
If `name` is None, yields all entry points in `group` from all | |
distributions in the working set, otherwise only ones matching | |
both `group` and `name` are yielded (in distribution order). | |
""" | |
return ( | |
entry | |
for dist in self | |
for entry in dist.get_entry_map(group).values() | |
if name is None or name == entry.name | |
) | |
def run_script(self, requires, script_name): | |
"""Locate distribution for `requires` and run `script_name` script""" | |
ns = sys._getframe(1).f_globals | |
name = ns['__name__'] | |
ns.clear() | |
ns['__name__'] = name | |
self.require(requires)[0].run_script(script_name, ns) | |
def __iter__(self): | |
"""Yield distributions for non-duplicate projects in the working set | |
The yield order is the order in which the items' path entries were | |
added to the working set. | |
""" | |
seen = {} | |
for item in self.entries: | |
if item not in self.entry_keys: | |
# workaround a cache issue | |
continue | |
for key in self.entry_keys[item]: | |
if key not in seen: | |
seen[key] = 1 | |
yield self.by_key[key] | |
def add(self, dist, entry=None, insert=True, replace=False): | |
"""Add `dist` to working set, associated with `entry` | |
If `entry` is unspecified, it defaults to the ``.location`` of `dist`. | |
On exit from this routine, `entry` is added to the end of the working | |
set's ``.entries`` (if it wasn't already present). | |
`dist` is only added to the working set if it's for a project that | |
doesn't already have a distribution in the set, unless `replace=True`. | |
If it's added, any callbacks registered with the ``subscribe()`` method | |
will be called. | |
""" | |
if insert: | |
dist.insert_on(self.entries, entry, replace=replace) | |
if entry is None: | |
entry = dist.location | |
keys = self.entry_keys.setdefault(entry, []) | |
keys2 = self.entry_keys.setdefault(dist.location, []) | |
if not replace and dist.key in self.by_key: | |
# ignore hidden distros | |
return | |
self.by_key[dist.key] = dist | |
if dist.key not in keys: | |
keys.append(dist.key) | |
if dist.key not in keys2: | |
keys2.append(dist.key) | |
self._added_new(dist) | |
def resolve(self, requirements, env=None, installer=None, | |
replace_conflicting=False, extras=None): | |
"""List all distributions needed to (recursively) meet `requirements` | |
`requirements` must be a sequence of ``Requirement`` objects. `env`, | |
if supplied, should be an ``Environment`` instance. If | |
not supplied, it defaults to all distributions available within any | |
entry or distribution in the working set. `installer`, if supplied, | |
will be invoked with each requirement that cannot be met by an | |
already-installed distribution; it should return a ``Distribution`` or | |
``None``. | |
Unless `replace_conflicting=True`, raises a VersionConflict exception | |
if | |
any requirements are found on the path that have the correct name but | |
the wrong version. Otherwise, if an `installer` is supplied it will be | |
invoked to obtain the correct version of the requirement and activate | |
it. | |
`extras` is a list of the extras to be used with these requirements. | |
This is important because extra requirements may look like `my_req; | |
extra = "my_extra"`, which would otherwise be interpreted as a purely | |
optional requirement. Instead, we want to be able to assert that these | |
requirements are truly required. | |
""" | |
# set up the stack | |
requirements = list(requirements)[::-1] | |
# set of processed requirements | |
processed = {} | |
# key -> dist | |
best = {} | |
to_activate = [] | |
req_extras = _ReqExtras() | |
# Mapping of requirement to set of distributions that required it; | |
# useful for reporting info about conflicts. | |
required_by = collections.defaultdict(set) | |
while requirements: | |
# process dependencies breadth-first | |
req = requirements.pop(0) | |
if req in processed: | |
# Ignore cyclic or redundant dependencies | |
continue | |
if not req_extras.markers_pass(req, extras): | |
continue | |
dist = best.get(req.key) | |
if dist is None: | |
# Find the best distribution and add it to the map | |
dist = self.by_key.get(req.key) | |
if dist is None or (dist not in req and replace_conflicting): | |
ws = self | |
if env is None: | |
if dist is None: | |
env = Environment(self.entries) | |
else: | |
# Use an empty environment and workingset to avoid | |
# any further conflicts with the conflicting | |
# distribution | |
env = Environment([]) | |
ws = WorkingSet([]) | |
dist = best[req.key] = env.best_match( | |
req, ws, installer, | |
replace_conflicting=replace_conflicting | |
) | |
if dist is None: | |
requirers = required_by.get(req, None) | |
raise DistributionNotFound(req, requirers) | |
to_activate.append(dist) | |
if dist not in req: | |
# Oops, the "best" so far conflicts with a dependency | |
dependent_req = required_by[req] | |
raise VersionConflict(dist, req).with_context(dependent_req) | |
# push the new requirements onto the stack | |
new_requirements = dist.requires(req.extras)[::-1] | |
requirements.extend(new_requirements) | |
# Register the new requirements needed by req | |
for new_requirement in new_requirements: | |
required_by[new_requirement].add(req.project_name) | |
req_extras[new_requirement] = req.extras | |
processed[req] = True | |
# return list of distros to activate | |
return to_activate | |
def find_plugins( | |
self, plugin_env, full_env=None, installer=None, fallback=True): | |
"""Find all activatable distributions in `plugin_env` | |
Example usage:: | |
distributions, errors = working_set.find_plugins( | |
Environment(plugin_dirlist) | |
) | |
# add plugins+libs to sys.path | |
map(working_set.add, distributions) | |
# display errors | |
print('Could not load', errors) | |
The `plugin_env` should be an ``Environment`` instance that contains | |
only distributions that are in the project's "plugin directory" or | |
directories. The `full_env`, if supplied, should be an ``Environment`` | |
contains all currently-available distributions. If `full_env` is not | |
supplied, one is created automatically from the ``WorkingSet`` this | |
method is called on, which will typically mean that every directory on | |
``sys.path`` will be scanned for distributions. | |
`installer` is a standard installer callback as used by the | |
``resolve()`` method. The `fallback` flag indicates whether we should | |
attempt to resolve older versions of a plugin if the newest version | |
cannot be resolved. | |
This method returns a 2-tuple: (`distributions`, `error_info`), where | |
`distributions` is a list of the distributions found in `plugin_env` | |
that were loadable, along with any other distributions that are needed | |
to resolve their dependencies. `error_info` is a dictionary mapping | |
unloadable plugin distributions to an exception instance describing the | |
error that occurred. Usually this will be a ``DistributionNotFound`` or | |
``VersionConflict`` instance. | |
""" | |
plugin_projects = list(plugin_env) | |
# scan project names in alphabetic order | |
plugin_projects.sort() | |
error_info = {} | |
distributions = {} | |
if full_env is None: | |
env = Environment(self.entries) | |
env += plugin_env | |
else: | |
env = full_env + plugin_env | |
shadow_set = self.__class__([]) | |
# put all our entries in shadow_set | |
list(map(shadow_set.add, self)) | |
for project_name in plugin_projects: | |
for dist in plugin_env[project_name]: | |
req = [dist.as_requirement()] | |
try: | |
resolvees = shadow_set.resolve(req, env, installer) | |
except ResolutionError as v: | |
# save error info | |
error_info[dist] = v | |
if fallback: | |
# try the next older version of project | |
continue | |
else: | |
# give up on this project, keep going | |
break | |
else: | |
list(map(shadow_set.add, resolvees)) | |
distributions.update(dict.fromkeys(resolvees)) | |
# success, no need to try any more versions of this project | |
break | |
distributions = list(distributions) | |
distributions.sort() | |
return distributions, error_info | |
def require(self, *requirements): | |
"""Ensure that distributions matching `requirements` are activated | |
`requirements` must be a string or a (possibly-nested) sequence | |
thereof, specifying the distributions and versions required. The | |
return value is a sequence of the distributions that needed to be | |
activated to fulfill the requirements; all relevant distributions are | |
included, even if they were already activated in this working set. | |
""" | |
needed = self.resolve(parse_requirements(requirements)) | |
for dist in needed: | |
self.add(dist) | |
return needed | |
def subscribe(self, callback, existing=True): | |
"""Invoke `callback` for all distributions | |
If `existing=True` (default), | |
call on all existing ones, as well. | |
""" | |
if callback in self.callbacks: | |
return | |
self.callbacks.append(callback) | |
if not existing: | |
return | |
for dist in self: | |
callback(dist) | |
def _added_new(self, dist): | |
for callback in self.callbacks: | |
callback(dist) | |
def __getstate__(self): | |
return ( | |
self.entries[:], self.entry_keys.copy(), self.by_key.copy(), | |
self.callbacks[:] | |
) | |
def __setstate__(self, e_k_b_c): | |
entries, keys, by_key, callbacks = e_k_b_c | |
self.entries = entries[:] | |
self.entry_keys = keys.copy() | |
self.by_key = by_key.copy() | |
self.callbacks = callbacks[:] | |
class _ReqExtras(dict): | |
""" | |
Map each requirement to the extras that demanded it. | |
""" | |
def markers_pass(self, req, extras=None): | |
""" | |
Evaluate markers for req against each extra that | |
demanded it. | |
Return False if the req has a marker and fails | |
evaluation. Otherwise, return True. | |
""" | |
extra_evals = ( | |
req.marker.evaluate({'extra': extra}) | |
for extra in self.get(req, ()) + (extras or (None,)) | |
) | |
return not req.marker or any(extra_evals) | |
class Environment: | |
"""Searchable snapshot of distributions on a search path""" | |
def __init__( | |
self, search_path=None, platform=get_supported_platform(), | |
python=PY_MAJOR): | |
"""Snapshot distributions available on a search path | |
Any distributions found on `search_path` are added to the environment. | |
`search_path` should be a sequence of ``sys.path`` items. If not | |
supplied, ``sys.path`` is used. | |
`platform` is an optional string specifying the name of the platform | |
that platform-specific distributions must be compatible with. If | |
unspecified, it defaults to the current platform. `python` is an | |
optional string naming the desired version of Python (e.g. ``'3.6'``); | |
it defaults to the current version. | |
You may explicitly set `platform` (and/or `python`) to ``None`` if you | |
wish to map *all* distributions, not just those compatible with the | |
running platform or Python version. | |
""" | |
self._distmap = {} | |
self.platform = platform | |
self.python = python | |
self.scan(search_path) | |
def can_add(self, dist): | |
"""Is distribution `dist` acceptable for this environment? | |
The distribution must match the platform and python version | |
requirements specified when this environment was created, or False | |
is returned. | |
""" | |
py_compat = ( | |
self.python is None | |
or dist.py_version is None | |
or dist.py_version == self.python | |
) | |
return py_compat and compatible_platforms(dist.platform, self.platform) | |
def remove(self, dist): | |
"""Remove `dist` from the environment""" | |
self._distmap[dist.key].remove(dist) | |
def scan(self, search_path=None): | |
"""Scan `search_path` for distributions usable in this environment | |
Any distributions found are added to the environment. | |
`search_path` should be a sequence of ``sys.path`` items. If not | |
supplied, ``sys.path`` is used. Only distributions conforming to | |
the platform/python version defined at initialization are added. | |
""" | |
if search_path is None: | |
search_path = sys.path | |
for item in search_path: | |
for dist in find_distributions(item): | |
self.add(dist) | |
def __getitem__(self, project_name): | |
"""Return a newest-to-oldest list of distributions for `project_name` | |
Uses case-insensitive `project_name` comparison, assuming all the | |
project's distributions use their project's name converted to all | |
lowercase as their key. | |
""" | |
distribution_key = project_name.lower() | |
return self._distmap.get(distribution_key, []) | |
def add(self, dist): | |
"""Add `dist` if we ``can_add()`` it and it has not already been added | |
""" | |
if self.can_add(dist) and dist.has_version(): | |
dists = self._distmap.setdefault(dist.key, []) | |
if dist not in dists: | |
dists.append(dist) | |
dists.sort(key=operator.attrgetter('hashcmp'), reverse=True) | |
def best_match( | |
self, req, working_set, installer=None, replace_conflicting=False): | |
"""Find distribution best matching `req` and usable on `working_set` | |
This calls the ``find(req)`` method of the `working_set` to see if a | |
suitable distribution is already active. (This may raise | |
``VersionConflict`` if an unsuitable version of the project is already | |
active in the specified `working_set`.) If a suitable distribution | |
isn't active, this method returns the newest distribution in the | |
environment that meets the ``Requirement`` in `req`. If no suitable | |
distribution is found, and `installer` is supplied, then the result of | |
calling the environment's ``obtain(req, installer)`` method will be | |
returned. | |
""" | |
try: | |
dist = working_set.find(req) | |
except VersionConflict: | |
if not replace_conflicting: | |
raise | |
dist = None | |
if dist is not None: | |
return dist | |
for dist in self[req.key]: | |
if dist in req: | |
return dist | |
# try to download/install | |
return self.obtain(req, installer) | |
def obtain(self, requirement, installer=None): | |
"""Obtain a distribution matching `requirement` (e.g. via download) | |
Obtain a distro that matches requirement (e.g. via download). In the | |
base ``Environment`` class, this routine just returns | |
``installer(requirement)``, unless `installer` is None, in which case | |
None is returned instead. This method is a hook that allows subclasses | |
to attempt other ways of obtaining a distribution before falling back | |
to the `installer` argument.""" | |
if installer is not None: | |
return installer(requirement) | |
def __iter__(self): | |
"""Yield the unique project names of the available distributions""" | |
for key in self._distmap.keys(): | |
if self[key]: | |
yield key | |
def __iadd__(self, other): | |
"""In-place addition of a distribution or environment""" | |
if isinstance(other, Distribution): | |
self.add(other) | |
elif isinstance(other, Environment): | |
for project in other: | |
for dist in other[project]: | |
self.add(dist) | |
else: | |
raise TypeError("Can't add %r to environment" % (other,)) | |
return self | |
def __add__(self, other): | |
"""Add an environment or distribution to an environment""" | |
new = self.__class__([], platform=None, python=None) | |
for env in self, other: | |
new += env | |
return new | |
# XXX backward compatibility | |
AvailableDistributions = Environment | |
class ExtractionError(RuntimeError): | |
"""An error occurred extracting a resource | |
The following attributes are available from instances of this exception: | |
manager | |
The resource manager that raised this exception | |
cache_path | |
The base directory for resource extraction | |
original_error | |
The exception instance that caused extraction to fail | |
""" | |
class ResourceManager: | |
"""Manage resource extraction and packages""" | |
extraction_path = None | |
def __init__(self): | |
self.cached_files = {} | |
def resource_exists(self, package_or_requirement, resource_name): | |
"""Does the named resource exist?""" | |
return get_provider(package_or_requirement).has_resource(resource_name) | |
def resource_isdir(self, package_or_requirement, resource_name): | |
"""Is the named resource an existing directory?""" | |
return get_provider(package_or_requirement).resource_isdir( | |
resource_name | |
) | |
def resource_filename(self, package_or_requirement, resource_name): | |
"""Return a true filesystem path for specified resource""" | |
return get_provider(package_or_requirement).get_resource_filename( | |
self, resource_name | |
) | |
def resource_stream(self, package_or_requirement, resource_name): | |
"""Return a readable file-like object for specified resource""" | |
return get_provider(package_or_requirement).get_resource_stream( | |
self, resource_name | |
) | |
def resource_string(self, package_or_requirement, resource_name): | |
"""Return specified resource as a string""" | |
return get_provider(package_or_requirement).get_resource_string( | |
self, resource_name | |
) | |
def resource_listdir(self, package_or_requirement, resource_name): | |
"""List the contents of the named resource directory""" | |
return get_provider(package_or_requirement).resource_listdir( | |
resource_name | |
) | |
def extraction_error(self): | |
"""Give an error message for problems extracting file(s)""" | |
old_exc = sys.exc_info()[1] | |
cache_path = self.extraction_path or get_default_cache() | |
tmpl = textwrap.dedent(""" | |
Can't extract file(s) to egg cache | |
The following error occurred while trying to extract file(s) | |
to the Python egg cache: | |
{old_exc} | |
The Python egg cache directory is currently set to: | |
{cache_path} | |
Perhaps your account does not have write access to this directory? | |
You can change the cache directory by setting the PYTHON_EGG_CACHE | |
environment variable to point to an accessible directory. | |
""").lstrip() | |
err = ExtractionError(tmpl.format(**locals())) | |
err.manager = self | |
err.cache_path = cache_path | |
err.original_error = old_exc | |
raise err | |
def get_cache_path(self, archive_name, names=()): | |
"""Return absolute location in cache for `archive_name` and `names` | |
The parent directory of the resulting path will be created if it does | |
not already exist. `archive_name` should be the base filename of the | |
enclosing egg (which may not be the name of the enclosing zipfile!), | |
including its ".egg" extension. `names`, if provided, should be a | |
sequence of path name parts "under" the egg's extraction location. | |
This method should only be called by resource providers that need to | |
obtain an extraction location, and only for names they intend to | |
extract, as it tracks the generated names for possible cleanup later. | |
""" | |
extract_path = self.extraction_path or get_default_cache() | |
target_path = os.path.join(extract_path, archive_name + '-tmp', *names) | |
try: | |
_bypass_ensure_directory(target_path) | |
except Exception: | |
self.extraction_error() | |
self._warn_unsafe_extraction_path(extract_path) | |
self.cached_files[target_path] = 1 | |
return target_path | |
@staticmethod | |
def _warn_unsafe_extraction_path(path): | |
""" | |
If the default extraction path is overridden and set to an insecure | |
location, such as /tmp, it opens up an opportunity for an attacker to | |
replace an extracted file with an unauthorized payload. Warn the user | |
if a known insecure location is used. | |
See Distribute #375 for more details. | |
""" | |
if os.name == 'nt' and not path.startswith(os.environ['windir']): | |
# On Windows, permissions are generally restrictive by default | |
# and temp directories are not writable by other users, so | |
# bypass the warning. | |
return | |
mode = os.stat(path).st_mode | |
if mode & stat.S_IWOTH or mode & stat.S_IWGRP: | |
msg = ( | |
"%s is writable by group/others and vulnerable to attack " | |
"when " | |
"used with get_resource_filename. Consider a more secure " | |
"location (set with .set_extraction_path or the " | |
"PYTHON_EGG_CACHE environment variable)." % path | |
) | |
warnings.warn(msg, UserWarning) | |
def postprocess(self, tempname, filename): | |
"""Perform any platform-specific postprocessing of `tempname` | |
This is where Mac header rewrites should be done; other platforms don't | |
have anything special they should do. | |
Resource providers should call this method ONLY after successfully | |
extracting a compressed resource. They must NOT call it on resources | |
that are already in the filesystem. | |
`tempname` is the current (temporary) name of the file, and `filename` | |
is the name it will be renamed to by the caller after this routine | |
returns. | |
""" | |
if os.name == 'posix': | |
# Make the resource executable | |
mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777 | |
os.chmod(tempname, mode) | |
def set_extraction_path(self, path): | |
"""Set the base path where resources will be extracted to, if needed. | |
If you do not call this routine before any extractions take place, the | |
path defaults to the return value of ``get_default_cache()``. (Which | |
is based on the ``PYTHON_EGG_CACHE`` environment variable, with various | |
platform-specific fallbacks. See that routine's documentation for more | |
details.) | |
Resources are extracted to subdirectories of this path based upon | |
information given by the ``IResourceProvider``. You may set this to a | |
temporary directory, but then you must call ``cleanup_resources()`` to | |
delete the extracted files when done. There is no guarantee that | |
``cleanup_resources()`` will be able to remove all extracted files. | |
(Note: you may not change the extraction path for a given resource | |
manager once resources have been extracted, unless you first call | |
``cleanup_resources()``.) | |
""" | |
if self.cached_files: | |
raise ValueError( | |
"Can't change extraction path, files already extracted" | |
) | |
self.extraction_path = path | |
def cleanup_resources(self, force=False): | |
""" | |
Delete all extracted resource files and directories, returning a list | |
of the file and directory names that could not be successfully removed. | |
This function does not have any concurrency protection, so it should | |
generally only be called when the extraction path is a temporary | |
directory exclusive to a single process. This method is not | |
automatically called; you must call it explicitly or register it as an | |
``atexit`` function if you wish to ensure cleanup of a temporary | |
directory used for extractions. | |
""" | |
# XXX | |
def get_default_cache(): | |
""" | |
Return the ``PYTHON_EGG_CACHE`` environment variable | |
or a platform-relevant user cache dir for an app | |
named "Python-Eggs". | |
""" | |
return ( | |
os.environ.get('PYTHON_EGG_CACHE') | |
or appdirs.user_cache_dir(appname='Python-Eggs') | |
) | |
def safe_name(name): | |
"""Convert an arbitrary string to a standard distribution name | |
Any runs of non-alphanumeric/. characters are replaced with a single '-'. | |
""" | |
return re.sub('[^A-Za-z0-9.]+', '-', name) | |
def safe_version(version): | |
""" | |
Convert an arbitrary string to a standard version string | |
""" | |
try: | |
# normalize the version | |
return str(packaging.version.Version(version)) | |
except packaging.version.InvalidVersion: | |
version = version.replace(' ', '.') | |
return re.sub('[^A-Za-z0-9.]+', '-', version) | |
def safe_extra(extra): | |
"""Convert an arbitrary string to a standard 'extra' name | |
Any runs of non-alphanumeric characters are replaced with a single '_', | |
and the result is always lowercased. | |
""" | |
return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() | |
def to_filename(name): | |
"""Convert a project or version name to its filename-escaped form | |
Any '-' characters are currently replaced with '_'. | |
""" | |
return name.replace('-', '_') | |
def invalid_marker(text): | |
""" | |
Validate text as a PEP 508 environment marker; return an exception | |
if invalid or False otherwise. | |
""" | |
try: | |
evaluate_marker(text) | |
except SyntaxError as e: | |
e.filename = None | |
e.lineno = None | |
return e | |
return False | |
def evaluate_marker(text, extra=None): | |
""" | |
Evaluate a PEP 508 environment marker. | |
Return a boolean indicating the marker result in this environment. | |
Raise SyntaxError if marker is invalid. | |
This implementation uses the 'pyparsing' module. | |
""" | |
try: | |
marker = packaging.markers.Marker(text) | |
return marker.evaluate() | |
except packaging.markers.InvalidMarker as e: | |
raise SyntaxError(e) | |
class NullProvider: | |
"""Try to implement resources and metadata for arbitrary PEP 302 loaders""" | |
egg_name = None | |
egg_info = None | |
loader = None | |
def __init__(self, module): | |
self.loader = getattr(module, '__loader__', None) | |
self.module_path = os.path.dirname(getattr(module, '__file__', '')) | |
def get_resource_filename(self, manager, resource_name): | |
return self._fn(self.module_path, resource_name) | |
def get_resource_stream(self, manager, resource_name): | |
return io.BytesIO(self.get_resource_string(manager, resource_name)) | |
def get_resource_string(self, manager, resource_name): | |
return self._get(self._fn(self.module_path, resource_name)) | |
def has_resource(self, resource_name): | |
return self._has(self._fn(self.module_path, resource_name)) | |
def _get_metadata_path(self, name): | |
return self._fn(self.egg_info, name) | |
def has_metadata(self, name): | |
if not self.egg_info: | |
return self.egg_info | |
path = self._get_metadata_path(name) | |
return self._has(path) | |
def get_metadata(self, name): | |
if not self.egg_info: | |
return "" | |
path = self._get_metadata_path(name) | |
value = self._get(path) | |
if six.PY2: | |
return value | |
try: | |
return value.decode('utf-8') | |
except UnicodeDecodeError as exc: | |
# Include the path in the error message to simplify | |
# troubleshooting, and without changing the exception type. | |
exc.reason += ' in {} file at path: {}'.format(name, path) | |
raise | |
def get_metadata_lines(self, name): | |
return yield_lines(self.get_metadata(name)) | |
def resource_isdir(self, resource_name): | |
return self._isdir(self._fn(self.module_path, resource_name)) | |
def metadata_isdir(self, name): | |
return self.egg_info and self._isdir(self._fn(self.egg_info, name)) | |
def resource_listdir(self, resource_name): | |
return self._listdir(self._fn(self.module_path, resource_name)) | |
def metadata_listdir(self, name): | |
if self.egg_info: | |
return self._listdir(self._fn(self.egg_info, name)) | |
return [] | |
def run_script(self, script_name, namespace): | |
script = 'scripts/' + script_name | |
if not self.has_metadata(script): | |
raise ResolutionError( | |
"Script {script!r} not found in metadata at {self.egg_info!r}" | |
.format(**locals()), | |
) | |
script_text = self.get_metadata(script).replace('\r\n', '\n') | |
script_text = script_text.replace('\r', '\n') | |
script_filename = self._fn(self.egg_info, script) | |
namespace['__file__'] = script_filename | |
if os.path.exists(script_filename): | |
source = open(script_filename).read() | |
code = compile(source, script_filename, 'exec') | |
exec(code, namespace, namespace) | |
else: | |
from linecache import cache | |
cache[script_filename] = ( | |
len(script_text), 0, script_text.split('\n'), script_filename | |
) | |
script_code = compile(script_text, script_filename, 'exec') | |
exec(script_code, namespace, namespace) | |
def _has(self, path): | |
raise NotImplementedError( | |
"Can't perform this operation for unregistered loader type" | |
) | |
def _isdir(self, path): | |
raise NotImplementedError( | |
"Can't perform this operation for unregistered loader type" | |
) | |
def _listdir(self, path): | |
raise NotImplementedError( | |
"Can't perform this operation for unregistered loader type" | |
) | |
def _fn(self, base, resource_name): | |
self._validate_resource_path(resource_name) | |
if resource_name: | |
return os.path.join(base, *resource_name.split('/')) | |
return base | |
@staticmethod | |
def _validate_resource_path(path): | |
""" | |
Validate the resource paths according to the docs. | |
https://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access | |
>>> warned = getfixture('recwarn') | |
>>> warnings.simplefilter('always') | |
>>> vrp = NullProvider._validate_resource_path | |
>>> vrp('foo/bar.txt') | |
>>> bool(warned) | |
False | |
>>> vrp('../foo/bar.txt') | |
>>> bool(warned) | |
True | |
>>> warned.clear() | |
>>> vrp('/foo/bar.txt') | |
>>> bool(warned) | |
True | |
>>> vrp('foo/../../bar.txt') | |
>>> bool(warned) | |
True | |
>>> warned.clear() | |
>>> vrp('foo/f../bar.txt') | |
>>> bool(warned) | |
False | |
Windows path separators are straight-up disallowed. | |
>>> vrp(r'\\foo/bar.txt') | |
Traceback (most recent call last): | |
... | |
ValueError: Use of .. or absolute path in a resource path \ | |
is not allowed. | |
>>> vrp(r'C:\\foo/bar.txt') | |
Traceback (most recent call last): | |
... | |
ValueError: Use of .. or absolute path in a resource path \ | |
is not allowed. | |
Blank values are allowed | |
>>> vrp('') | |
>>> bool(warned) | |
False | |
Non-string values are not. | |
>>> vrp(None) | |
Traceback (most recent call last): | |
... | |
AttributeError: ... | |
""" | |
invalid = ( | |
os.path.pardir in path.split(posixpath.sep) or | |
posixpath.isabs(path) or | |
ntpath.isabs(path) | |
) | |
if not invalid: | |
return | |
msg = "Use of .. or absolute path in a resource path is not allowed." | |
# Aggressively disallow Windows absolute paths | |
if ntpath.isabs(path) and not posixpath.isabs(path): | |
raise ValueError(msg) | |
# for compatibility, warn; in future | |
# raise ValueError(msg) | |
warnings.warn( | |
msg[:-1] + " and will raise exceptions in a future release.", | |
DeprecationWarning, | |
stacklevel=4, | |
) | |
def _get(self, path): | |
if hasattr(self.loader, 'get_data'): | |
return self.loader.get_data(path) | |
raise NotImplementedError( | |
"Can't perform this operation for loaders without 'get_data()'" | |
) | |
register_loader_type(object, NullProvider) | |
class EggProvider(NullProvider): | |
"""Provider based on a virtual filesystem""" | |
def __init__(self, module): | |
NullProvider.__init__(self, module) | |
self._setup_prefix() | |
def _setup_prefix(self): | |
# we assume here that our metadata may be nested inside a "basket" | |
# of multiple eggs; that's why we use module_path instead of .archive | |
path = self.module_path | |
old = None | |
while path != old: | |
if _is_egg_path(path): | |
self.egg_name = os.path.basename(path) | |
self.egg_info = os.path.join(path, 'EGG-INFO') | |
self.egg_root = path | |
break | |
old = path | |
path, base = os.path.split(path) | |
class DefaultProvider(EggProvider): | |
"""Provides access to package resources in the filesystem""" | |
def _has(self, path): | |
return os.path.exists(path) | |
def _isdir(self, path): | |
return os.path.isdir(path) | |
def _listdir(self, path): | |
return os.listdir(path) | |
def get_resource_stream(self, manager, resource_name): | |
return open(self._fn(self.module_path, resource_name), 'rb') | |
def _get(self, path): | |
with open(path, 'rb') as stream: | |
return stream.read() | |
@classmethod | |
def _register(cls): | |
loader_names = 'SourceFileLoader', 'SourcelessFileLoader', | |
for name in loader_names: | |
loader_cls = getattr(importlib_machinery, name, type(None)) | |
register_loader_type(loader_cls, cls) | |
DefaultProvider._register() | |
class EmptyProvider(NullProvider): | |
"""Provider that returns nothing for all requests""" | |
module_path = None | |
_isdir = _has = lambda self, path: False | |
def _get(self, path): | |
return '' | |
def _listdir(self, path): | |
return [] | |
def __init__(self): | |
pass | |
empty_provider = EmptyProvider() | |
class ZipManifests(dict): | |
""" | |
zip manifest builder | |
""" | |
@classmethod | |
def build(cls, path): | |
""" | |
Build a dictionary similar to the zipimport directory | |
caches, except instead of tuples, store ZipInfo objects. | |
Use a platform-specific path separator (os.sep) for the path keys | |
for compatibility with pypy on Windows. | |
""" | |
with zipfile.ZipFile(path) as zfile: | |
items = ( | |
( | |
name.replace('/', os.sep), | |
zfile.getinfo(name), | |
) | |
for name in zfile.namelist() | |
) | |
return dict(items) | |
load = build | |
class MemoizedZipManifests(ZipManifests): | |
""" | |
Memoized zipfile manifests. | |
""" | |
manifest_mod = collections.namedtuple('manifest_mod', 'manifest mtime') | |
def load(self, path): | |
""" | |
Load a manifest at path or return a suitable manifest already loaded. | |
""" | |
path = os.path.normpath(path) | |
mtime = os.stat(path).st_mtime | |
if path not in self or self[path].mtime != mtime: | |
manifest = self.build(path) | |
self[path] = self.manifest_mod(manifest, mtime) | |
return self[path].manifest | |
class ZipProvider(EggProvider): | |
"""Resource support for zips and eggs""" | |
eagers = None | |
_zip_manifests = MemoizedZipManifests() | |
def __init__(self, module): | |
EggProvider.__init__(self, module) | |
self.zip_pre = self.loader.archive + os.sep | |
def _zipinfo_name(self, fspath): | |
# Convert a virtual filename (full path to file) into a zipfile subpath | |
# usable with the zipimport directory cache for our target archive | |
fspath = fspath.rstrip(os.sep) | |
if fspath == self.loader.archive: | |
return '' | |
if fspath.startswith(self.zip_pre): | |
return fspath[len(self.zip_pre):] | |
raise AssertionError( | |
"%s is not a subpath of %s" % (fspath, self.zip_pre) | |
) | |
def _parts(self, zip_path): | |
# Convert a zipfile subpath into an egg-relative path part list. | |
# pseudo-fs path | |
fspath = self.zip_pre + zip_path | |
if fspath.startswith(self.egg_root + os.sep): | |
return fspath[len(self.egg_root) + 1:].split(os.sep) | |
raise AssertionError( | |
"%s is not a subpath of %s" % (fspath, self.egg_root) | |
) | |
@property | |
def zipinfo(self): | |
return self._zip_manifests.load(self.loader.archive) | |
def get_resource_filename(self, manager, resource_name): | |
if not self.egg_name: | |
raise NotImplementedError( | |
"resource_filename() only supported for .egg, not .zip" | |
) | |
# no need to lock for extraction, since we use temp names | |
zip_path = self._resource_to_zip(resource_name) | |
eagers = self._get_eager_resources() | |
if '/'.join(self._parts(zip_path)) in eagers: | |
for name in eagers: | |
self._extract_resource(manager, self._eager_to_zip(name)) | |
return self._extract_resource(manager, zip_path) | |
@staticmethod | |
def _get_date_and_size(zip_stat): | |
size = zip_stat.file_size | |
# ymdhms+wday, yday, dst | |
date_time = zip_stat.date_time + (0, 0, -1) | |
# 1980 offset already done | |
timestamp = time.mktime(date_time) | |
return timestamp, size | |
def _extract_resource(self, manager, zip_path): | |
if zip_path in self._index(): | |
for name in self._index()[zip_path]: | |
last = self._extract_resource( | |
manager, os.path.join(zip_path, name) | |
) | |
# return the extracted directory name | |
return os.path.dirname(last) | |
timestamp, size = self._get_date_and_size(self.zipinfo[zip_path]) | |
if not WRITE_SUPPORT: | |
raise IOError('"os.rename" and "os.unlink" are not supported ' | |
'on this platform') | |
try: | |
real_path = manager.get_cache_path( | |
self.egg_name, self._parts(zip_path) | |
) | |
if self._is_current(r |
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)
(Sorry about that, but we can’t show files that are this big right now.)