Sync Pipfile and setup.py

sync_pipfile_setup.py

# Sync Pipfile with setup.py dependencies
# Assumptions:
# - You are running in a directory with Pipfile, Pipfile.lock & setup.py
# - Your setup.py calls a function named setup()
# - setup() is called with keyword arguments of install_requires and dependency_links (can be empty lists)
# - All your remote dependencies are HTTPS git

import pipfile
import ast
import json
import astunparse
from yapf.yapflib import yapf_api
from typing import Union

pipfile_ = pipfile.load()

with open('Pipfile.lock') as file:
    pipfile_lock = json.load(file)


def format_str_config(name: str, config: str) -> str:
    name = name.lower()
    if name not in pipfile_lock['default'] and name.replace(
            '_', '-') in pipfile_lock['default']:
        name = name.replace('_', '-')
    if config == '*':
        return f'{pipfile_lock["default"][name]["version"]}'
    return config


def format_dict_config(name: str, config: dict) -> str:
    formatted = ''
    for key, value in config.items():
        if key == 'version':
            formatted += format_str_config(name, value)
        if key == 'os_name':
            formatted += f'; os_name{value}'
    return formatted


def is_editable(config: Union[str, dict]) -> bool:
    return isinstance(config, dict) and 'editable' in config


class UpdateInstallRequires(ast.NodeTransformer):
    def visit_Call(self, node):
        # TODO stronger check
        if node.func.id == 'setup':
            for keyword in node.keywords:
                if keyword.arg == 'install_requires':
                    install_requires = keyword.value
                if keyword.arg == 'dependency_links':
                    dependency_links = keyword.value
        dependency_links.elts = [
            ast.Str(f'{config["git"]}#egg={name}')
            for name, config in pipfile_.data['default'].items()
            if is_editable(config)
        ]
        install_requires.elts = [
            ast.Str(
                f'{name}{format_str_config(name, config) if isinstance(config, str) else format_dict_config(name, config)}'
            ) if not is_editable(config) else ast.Str(name)
            for name, config in pipfile_.data['default'].items()
        ]
        return node


with open('setup.py') as file:
    tree = ast.parse(file.read())

tree = UpdateInstallRequires().visit(tree)
source = astunparse.unparse(tree)
reformatted_source, _ = yapf_api.FormatCode(source)

with open('setup.py', 'w+') as file:
    file.write(reformatted_source)

I like this script a lot, but I'm getting a KeyError: 'setuptools'... which i suspect is from my from setuptools import setup.

Any suggestions on how to fix this?

Full stack dump

(pyclibase-tiNtbtXg) bash-4.4$ python sync_pipfile_setup.py
Traceback (most recent call last):
  File "sync_pipfile_setup.py", line 81, in <module>
    tree = UpdateInstallRequires().visit(tree)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ast.py", line 253, in visit
    return visitor(node)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ast.py", line 308, in generic_visit
    value = self.visit(value)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ast.py", line 253, in visit
    return visitor(node)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ast.py", line 317, in generic_visit
    new_node = self.visit(old_value)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/ast.py", line 253, in visit
    return visitor(node)
  File "sync_pipfile_setup.py", line 73, in visit_Call
    for name, config in pipfile_.data['default'].items()
  File "sync_pipfile_setup.py", line 73, in <listcomp>
    for name, config in pipfile_.data['default'].items()
  File "sync_pipfile_setup.py", line 32, in format_str_config
    return f'{pipfile_lock["default"][name]["version"]}'
KeyError: 'setuptools'
(pyclibase-tiNtbtXg) bash-4.4$

Hi! I developed a package specifically for this: pipenv-setup. which does the sync on one command $ pipenv-setup sync.

My packages refined the following:

• This script strips all comments away. pipenv-setup keeps them and edit install_requires, dependency_links in-place or create them when they are absent.
• More general markers including "os_name" this script has.
• General package config support. For example VCS other than git, built distribution with url.
• Bug fixes. For example this code doesn't work when there exist function calls other than setup()
• Black for formatting

And thanks for your inspiration! It saved me a lot of time!!

@bradwood
The traceback shows that package setuptools is not in pipfile.lock
maybe just run pipenv update to make sure it goes in the file

@Madoshakalaka glad it was useful