Skip to content

Instantly share code, notes, and snippets.

@dfee
Created April 27, 2018 01:16
Show Gist options
  • Save dfee/cc8438f110edbae0c933140d6fa0ff75 to your computer and use it in GitHub Desktop.
Save dfee/cc8438f110edbae0c933140d6fa0ff75 to your computer and use it in GitHub Desktop.
manifest_vk: convert **kwargs to explicit keyword arguments

Python: make explicit variable keywords in a function's signature

Python functions can take a variable keyword parameter, often known as **kwargs.

def myfunc(**kwargs):
    print(kwargs)

> myfunc(a=1, b=2)
{'a': 1, 'b': 2}

Sometimes you want to specify precisely which keyword arguments exist.

class User:
    def __init__(self, email_address=None, password=None, first_name=None, last_name=None):
        self.email_address = email_address
        self.password = password
        self.first_name = first_name
        self.last_name = last_name
    
    def __repr__(self):
        name = f'{type(self).__name__}'
        attrs = ', '.join([
            f'{k}={v}' for k, v in self.__dict__.items() 
            if not k.startswith('_')
        ])
        return f"<{name}: {attrs}>"
  
def create_user(*, email_address, password, first_name=None, last_name=None):
    return User(
        email_address=email_address,
        password=password,
        first_name=first_name,
        last_name=last_name,
    )

> user = create_user(
    email_address='[email protected]',
    password='dummy',
    first_name='Devin',
    last_name='Fee',
)
> print(user)
<User: email_address=None, password=None, first_name=None, last_name=None>

Which works great, until you want to distinguish between a provided keyword argument, and a possible keyword argument. This requires a marker, e.g. Missing.

class _Missing:
    pass
Missing = _Missing()

provided = lambda **kwargs: {k: v for k, v in kwargs.items() if v is not Missing}

def update_user(user, *, email_address=Missing, password=Missing, first_name=Missing, last_name=Missing):
    kwargs = provided(
        email_address=email_address,
        password=password,
        first_name=first_name,
        last_name=last_name,
    )
    for k, v in kwargs.items():
        setattr(user, k, v)
    return user

> user = update_user(user, password='seekrit')
> print(user)
<User: email_address=me@email.com, password=seekrit, first_name=Devin, last_name=Fee>

But what if you have many objects like User, and you want a base class to provide create / update functionality? Also, while the keyword arguments are great for documenation, they are tedious to write out in the signature of a function and then recombine in the function itself. This is what manifest_kw does for you - it converts the variable keyword argument (often, **kwargs) into explicit required, optional, and default keyword only arguments. Still want to provide **extra_kwargs? Just supply var_keywords=True to manifest_kw().

class BaseService:
    __model__ = None

    def create(self, **kwargs):
        # pylint: disable=E1102, not-callable
        return self.__model__(**kwargs)


class UserService(BaseService):
    __model__ = User

    @manifest_vk(
        required=('email_address', 'password'),
        optional=('first_name', 'last_name'),
    )
    def create(self, **kwargs):
        return super().create(**kwargs)

    def create2(
            self,
            *,
            email_address,
            password,
            first_name=Missing,
            last_name=Missing,
        ):
        return super().create(**provided(
            email_address=email_address,
            password=password,
            first_name=first_name,
            last_name=last_name,
        ))

import inspect
assert inspect.signature(UserService.create) == inspect.signature(UserService.create2)

This is beneficial for three reasons:

  1. it simplifies documentation and discoverability
  2. it reduces the amount of boilerplate needed to be explicit
  3. it guards against unwanted keyword arguments and allows providing defaults.

For the third point, consider the case where changing the email_address is a special revision and has side effects that are important to manage separately:

class BaseService:
    __model__ = None

    def create(self, **kwargs):
        # pylint: disable=E1102, not-callable
        return self.__model__(**kwargs)

  
class UserService(BaseService):
    __model__ = User

    @manifest_vk(
        required=('email_address', 'password'),
        optional=('first_name', 'last_name'),
    )
    def create(self, **kwargs):
        return super().create(**kwargs)

    @manifest_vk(
        optional=('first_name', 'last_name', 'password'),
    )
    def update(self, user, **kwargs):
        # no email_address here!
        for k, v in kwargs.items():
            setattr(user, k, v)
        return user
    
    def change_email_address(self, user, email_address):
        user.email_address = email_address
        send_confirmation_email()
        return user
import inspect
import pytest
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
import signature
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = sa.Column(sa.Integer, primary_key=True)
first_name = sa.Column(sa.String, primary_key=True)
last_name = sa.Column(sa.String, primary_key=True)
email_address = sa.Column(sa.String, primary_key=True)
password = sa.Column(sa.String, primary_key=True) # ;)
class BaseService:
__model__ = None
def create(self, **kwargs):
# pylint: disable=E1102, not-callable
return self.__model__(**kwargs)
class UserService(BaseService):
__model__ = User
@signature.manifest_vk(
required=('email_address', 'password'),
optional=('first_name', 'last_name'),
)
def create(self, **kwargs):
return super().create(**kwargs)
def create2(
self,
*,
email_address,
password,
first_name=signature.Missing,
last_name=signature.Missing,
):
return super().create(**signature.provided(
email_address=email_address,
password=password,
first_name=first_name,
last_name=last_name,
))
class TestUserService:
@pytest.fixture
def service(self):
return UserService()
def test_sig_match(self):
assert inspect.signature(UserService.create) == \
inspect.signature(UserService.create2)
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(email_address='[email protected]', password='dummy'),
None,
id='required_kwargs',
),
pytest.param(
dict(email_address='[email protected]'),
TypeError("create() missing a required argument: 'password'"),
id='missing_required_kwargs',
),
pytest.param(
dict(email_address='[email protected]', password='dummy', first_name='Jack'),
None,
id='required_and_optional_kwargs',
),
])
def test_create(self, service, kwargs, exc):
if exc:
with pytest.raises(type(exc)) as excinfo:
service.create(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
return
ins = service.create(**kwargs)
assert isinstance(ins, User)
for k, v in kwargs.items():
assert getattr(ins, k) == v
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(email_address='[email protected]', password='dummy'),
None,
id='required_kwargs',
),
pytest.param(
dict(email_address='[email protected]'),
TypeError(
"create2() missing 1 required keyword-only argument: "
"'password'"
),
id='missing_required_kwargs',
),
pytest.param(
dict(email_address='[email protected]', password='dummy', first_name='Jack'),
None,
id='required_and_optional_kwargs',
),
])
def test_create2(self, service, kwargs, exc):
if exc:
with pytest.raises(type(exc)) as excinfo:
service.create2(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
return
ins = service.create2(**kwargs)
assert isinstance(ins, User)
for k, v in kwargs.items():
assert getattr(ins, k) == v
import inspect
from functools import wraps
import pytest
class _Missing:
def __repr__(self):
return '<Missing>'
Missing = _Missing()
provided = lambda **kwargs: \
{k: v for k, v in kwargs.items() if v is not Missing}
def manifest_vk(
*,
required=None,
optional=None,
var_keywords=False,
**defaults,
):
'''
Creates a wrapper that provides a validating signature over a user-supplied
function or method.
:param required: an iterable of strings for generating signature parameters
without a default value.
:param optional: an iterable of strings for generating signature parameters
without a default value, but not required (default value is `Missing`).
:param var_keywords: whether the new signature allows variable keyword
arguments, perhaps for further extending.
:param defaults: a mapping of names to default values used for generating
signature parameters with a default value.
Usage:
>>> @manifest_vk(
... required=('req_1',),
... optional=('opt_1', 'opt_2',),
... var_keywords=True,
... def_1=None,
... )
>>> def myfunc(**kwargs):
... pass
>>> help(myfunc)
Help on function myfunc in module __main__:
myfunc(*, req_1, opt_1=<Missing>, opt_2=<Missing>, def_1=None, **kwargs)
'''
# pylint: disable=W0212, protected-access
def wrapper(func):
sig = inspect.signature(func)
vk_param = None
params = []
for param in sig.parameters.values():
if param.kind is inspect._VAR_KEYWORD:
vk_param = param
continue
params.append(param)
if not vk_param:
raise TypeError(
f"{func.__name__} doesn't specify a variable keyword parameter"
)
build_param = lambda name, default=inspect._empty: \
inspect.Parameter(
name=name,
kind=inspect._KEYWORD_ONLY,
default=default,
)
params.extend([build_param(name) for name in required or ()])
params.extend([build_param(name, Missing) for name in optional or ()])
params.extend([build_param(k, v) for k, v in defaults.items()])
if var_keywords:
params.append(vk_param)
@wraps(func)
def inner(*args, **kwargs):
try:
bound = inner.__signature__.bind(*args, **kwargs)
except TypeError as exc:
raise TypeError(f'{inner.__name__}() {exc.args[0]}')
bound.apply_defaults()
kwargs = provided(**bound.kwargs)
return func(*args, **kwargs)
inner.__signature__ = sig.replace(parameters=params)
return inner
return wrapper
def assert_signatures_match(func1, func2):
assert inspect.signature(func1) == inspect.signature(func2)
class TestService:
# pylint: disable=R0201, no-self-use
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(),
None,
id='expected_kwargs',
),
pytest.param(
dict(dummy=True),
TypeError("myfunc() got an unexpected keyword argument 'dummy'"),
id='unexpected_kwargs',
),
])
def test_empty(self, kwargs, exc):
@manifest_vk()
def myfunc(**kwargs):
return kwargs
assert_signatures_match(myfunc, lambda: None)
if not exc:
assert myfunc(**kwargs) == kwargs
return
with pytest.raises(type(exc)) as excinfo:
myfunc(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(id=1),
None,
id='expected_kwargs',
),
pytest.param(
dict(),
TypeError("myfunc() missing a required argument: 'id'"),
id='missing_kwargs',
),
pytest.param(
dict(id=1, dummy=True),
TypeError("myfunc() got an unexpected keyword argument 'dummy'"),
id='unexpected_kwargs',
),
])
def test_required(self, kwargs, exc):
@manifest_vk(required=['id'])
def myfunc(**kwargs):
return kwargs
assert_signatures_match(myfunc, lambda *, id: None)
if not exc:
assert myfunc(**kwargs) == kwargs
return
with pytest.raises(type(exc)) as excinfo:
myfunc(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(id=1),
None,
id='expected_kwargs',
),
pytest.param(
dict(),
None,
id='missing_kwargs',
),
pytest.param(
dict(id=1, dummy=True),
TypeError("myfunc() got an unexpected keyword argument 'dummy'"),
id='unexpected_kwargs',
),
])
def test_optional(self, kwargs, exc):
@manifest_vk(optional=['id'])
def myfunc(**kwargs):
return kwargs
assert_signatures_match(myfunc, lambda *, id=Missing: None)
if not exc:
assert myfunc(**kwargs) == kwargs
return
with pytest.raises(type(exc)) as excinfo:
myfunc(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
@pytest.mark.parametrize(('kwargs', 'exc'), [
pytest.param(
dict(id=1),
None,
id='expected_kwargs',
),
pytest.param(
dict(),
None,
id='missing_kwargs',
),
pytest.param(
dict(id=1, dummy=True),
TypeError("myfunc() got an unexpected keyword argument 'dummy'"),
id='unexpected_kwargs',
),
])
def test_defaults(self, kwargs, exc):
@manifest_vk(id=0)
def myfunc(**kwargs):
return kwargs
assert_signatures_match(myfunc, lambda *, id=0: None)
if not exc:
assert myfunc(**kwargs) == (kwargs or dict(id=0))
return
with pytest.raises(type(exc)) as excinfo:
myfunc(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
@pytest.mark.parametrize(('var_keywords', 'exc'), [
pytest.param(
True,
None,
id='allowed_unexpected_kwargs',
),
pytest.param(
False,
TypeError("myfunc() got an unexpected keyword argument 'dummy'"),
id='unallowed_unexpected_kwargs',
),
])
def test_var_keywords(self, var_keywords, exc):
@manifest_vk(var_keywords=var_keywords)
def myfunc(**kwargs):
return kwargs
kwargs = dict(dummy=True)
if not exc:
assert_signatures_match(myfunc, lambda **kwargs: None)
assert myfunc(**kwargs) == (kwargs or dict(id=0))
return
assert_signatures_match(myfunc, lambda: None)
with pytest.raises(type(exc)) as excinfo:
myfunc(**kwargs)
assert excinfo.value.args[0] == exc.args[0]
def test_missing_var_keyword(self):
with pytest.raises(TypeError) as excinfo:
# pylint: disable=W0612, unused-variable
@manifest_vk()
def myfunc():
pass
assert excinfo.value.args[0] == \
"myfunc doesn't specify a variable keyword parameter"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment