TypedDicts and Dicts in mypy

mypy-example.py

#! /usr/bin/env python3

"""TypedDicts and Dicts in mypy"""

from typing import TypedDict, Dict, Any, Mapping


JsonDict = Dict[str, Any]
JsonMapping = Mapping[str, Any]


# I have a generic function that expects to consume a JSON object as a dict.
def process_generic_dict(d: JsonDict) -> None:
    pass


# Within a specific context e.g. module, I know that I'm going to be using
# a specific shape of dictionary. An example I had when typing a background
# process in Synapse:
class Progress(TypedDict, total=False):
    """A dictionary that's either empty or holds a count of work `remaining`."""
    remaining: int


# I want to mark a function as accepting this shape of dict, and have that
# funtion pass off to a generic handle.  But this makes mypy sad:
# `incompatible type "Progress"; expected "Dict[str, Any]" [arg-type]`
def process_specific_dict(d: Progress) -> None:
    return process_generic_dict(d)


# This is a design choice of the mypy authors, see
# https://github.com/python/mypy/issues/4976
# I think the point is that if I hand off a Progress to a function that accepts
# a dict, I can't be sure that I get a Progress back again (the function could
# remove required keys). So it's not typesafe.

# Fair enough---there's no way to express mutability in the type system.
# ... or is there? Apparently `Mapping[T]` doesn't expose mutable methods, so
# we can do the following
def process_generic_mapping(d: JsonMapping) -> None:
    pass


def process_specific_mapping(d: Progress) -> None:
    return process_generic_mapping(d)


# But I don't think this is applicable to my use case. I want to write a function


async def _populate_user_directory_process_rooms(
    self, progress: Progress, batch_size: int
) -> int:
    pass


# and pass that function as the third argument of


def register_background_update_handler(
    self,
    update_name: str,
    update_handler: Callable[[JsonDict, int], Awaitable[int]],
):


# The update_handler will mutate the JsonDict it receives. So I can't replace
# JsonDict with JsonMap here. I think this means that I'll have to
# give up on using TypedDict for now.
# Other ideas:
#    do something clever with generics or overloads?
#    replace JsonDict with Any? Feels like a step back for marginal gain.
#    dataclasses rather than TypedDict? 

# Aside: it's a shame mypy can't say _why_ two types are incompatible in its
# output. I think that would have helped avoid some of the frustration on that
# issue.