Cloudpickle+Pydantic

mod.py

from fastapi import FastAPI
from pydantic import BaseModel

class User(BaseModel):
    name: str

user = User(name='a')


app = FastAPI()

@app.get("/")
def h(u: User) -> str:
    return "a"

def closure():
    app = FastAPI()

    @app.get("/")
    def h(u: User) -> str:
        return "a"

    return app

result.txt

============================= test session starts ==============================
platform darwin -- Python 3.6.9, pytest-5.3.5, py-1.10.0, pluggy-0.13.1 -- /Users/simonmo/miniconda3/bin/python
cachedir: .pytest_cache
benchmark: 3.2.2 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /private/tmp/t/test-pydantic
plugins: sugar-0.9.2, rerunfailures-9.1.1, asyncio-0.10.0, custom-exit-code-0.3.0, benchmark-3.2.2, aiohttp-0.3.0, timeout-1.4.2, cov-2.8.1
collecting ... collected 14 items

test_pydantic.py::test_serialize_cls FAILED                              [  7%]
test_pydantic.py::test_serialize_instance FAILED                         [ 14%]
test_pydantic.py::test_serialize_imported_cls PASSED                     [ 21%]
test_pydantic.py::test_serialize_imported_instance PASSED                [ 28%]
test_pydantic.py::test_serialize_app_no_route FAILED                     [ 35%]
test_pydantic.py::test_serialize_app_no_validation FAILED                [ 42%]
test_pydantic.py::test_serialize_app_primitive_type FAILED               [ 50%]
test_pydantic.py::test_serialize_app_pydantic_type_imported FAILED       [ 57%]
test_pydantic.py::test_serialize_app_pydantic_type_inline FAILED         [ 64%]
test_pydantic.py::test_serialize_app_imported FAILED                     [ 71%]
test_pydantic.py::test_serialize_app_pydantic_type_closure_ref FAILED    [ 78%]
test_pydantic.py::test_serialize_app_pydantic_type_closure_ref_import PASSED [ 85%]
test_pydantic.py::test_serialize_app_pydantic_type_closure PASSED        [ 92%]
test_pydantic.py::test_serialize_app_imported_closure PASSED             [100%]

=================================== FAILURES ===================================
______________________________ test_serialize_cls ______________________________

    def test_serialize_cls():
        class User(BaseModel):
            name: str
    
>       pickle.loads(pickle.dumps(User))

test_pydantic.py:9: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc108>
obj = <class 'test_pydantic.test_serialize_cls.<locals>.User'>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff21867a5c0>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
___________________________ test_serialize_instance ____________________________

    def test_serialize_instance():
        class User(BaseModel):
            name: str
    
>       pickle.loads(pickle.dumps(User(name='a')))

test_pydantic.py:16: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc288>
obj = User(name='a')

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff21867a750>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
_________________________ test_serialize_app_no_route __________________________

    def test_serialize_app_no_route():
        app = FastAPI()
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:32: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
E   RecursionError: maximum recursion depth exceeded while calling a Python object
!!! Recursion detected (same locals & position)
_______________________ test_serialize_app_no_validation _______________________

    def test_serialize_app_no_validation():
        app = FastAPI()
        @app.get("/")
        def hello() -> str:
            return "hi"
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:39: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
/Users/simonmo/miniconda3/lib/python3.6/site-packages/starlette/datastructures.py:669: in __getattr__
    return self._state[key]
E   RecursionError: maximum recursion depth exceeded while calling a Python object
!!! Recursion detected (same locals & position)
______________________ test_serialize_app_primitive_type _______________________

    def test_serialize_app_primitive_type():
        app = FastAPI()
        @app.get("/")
        def hello(v: str) -> str:
            return "hi"
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:47: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc4c8>
obj = <fastapi.applications.FastAPI object at 0x7ff1f90d3cf8>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff1f90b2110>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
__________________ test_serialize_app_pydantic_type_imported ___________________

    def test_serialize_app_pydantic_type_imported():
        from mod import User
    
        app = FastAPI()
        @app.get("/")
        def hello(v: str, u: User) -> str:
            return "hi"
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc648>
obj = <fastapi.applications.FastAPI object at 0x7ff1f9085080>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff1f90b24f8>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
___________________ test_serialize_app_pydantic_type_inline ____________________

    def test_serialize_app_pydantic_type_inline():
        class User(BaseModel):
            name: str
    
        app = FastAPI()
        @app.get("/")
        def hello(v: str, u: User) -> str:
            return "hi"
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:68: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc7c8>
obj = <fastapi.applications.FastAPI object at 0x7ff1f90a3da0>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff1f90b2b38>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
_________________________ test_serialize_app_imported __________________________

    def test_serialize_app_imported():
        from mod import app
>       pickle.loads(pickle.dumps(app))

test_pydantic.py:72: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bc888>
obj = <fastapi.applications.FastAPI object at 0x7ff21871e748>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction BaseModel.validate at 0x7ff21867ae58>: attribute lookup lambda12 on pydantic.main failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
_________________ test_serialize_app_pydantic_type_closure_ref _________________

    def test_serialize_app_pydantic_type_closure_ref():
        class User(BaseModel):
            name: str
        def make():
            app = FastAPI()
            @app.get("/")
            def hello(v: str, u: User) -> str:
                return "hi"
            return app
>       pickle.loads(pickle.dumps(make))

test_pydantic.py:84: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:73: in dumps
    cp.dump(obj)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ray.cloudpickle.cloudpickle_fast.CloudPickler object at 0x7ff2186bca08>
obj = <function test_serialize_app_pydantic_type_closure_ref.<locals>.make at 0x7ff1f9113510>

    def dump(self, obj):
        try:
>           return Pickler.dump(self, obj)
E           _pickle.PicklingError: Can't pickle <cyfunction str_validator at 0x7ff1f91091d8>: attribute lookup lambda12 on pydantic.validators failed

/Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle_fast.py:563: PicklingError
=============================== warnings summary ===============================
test_pydantic.py::test_serialize_instance
test_pydantic.py::test_serialize_imported_instance
  /Users/simonmo/Desktop/ray/ray/python/ray/cloudpickle/cloudpickle.py:469: DeprecationWarning: `__values__` attribute is deprecated, use `__dict__` instead
    (getattr(obj, '__values__', None) is not None)

-- Docs: https://docs.pytest.org/en/latest/warnings.html
=================== 9 failed, 5 passed, 2 warnings in 0.60s ====================

test_pydantic.py

from pydantic import BaseModel
from fastapi import FastAPI
import ray.cloudpickle as pickle

def test_serialize_cls():
    class User(BaseModel):
        name: str

    pickle.loads(pickle.dumps(User))


def test_serialize_instance():
    class User(BaseModel):
        name: str

    pickle.loads(pickle.dumps(User(name='a')))

def test_serialize_imported_cls():
    from mod import User

    pickle.loads(pickle.dumps(User))


def test_serialize_imported_instance():
    from mod import user

    pickle.loads(pickle.dumps(user))


def test_serialize_app_no_route():
    app = FastAPI()
    pickle.loads(pickle.dumps(app))

def test_serialize_app_no_validation():
    app = FastAPI()
    @app.get("/")
    def hello() -> str:
        return "hi"
    pickle.loads(pickle.dumps(app))


def test_serialize_app_primitive_type():
    app = FastAPI()
    @app.get("/")
    def hello(v: str) -> str:
        return "hi"
    pickle.loads(pickle.dumps(app))


def test_serialize_app_pydantic_type_imported():
    from mod import User

    app = FastAPI()
    @app.get("/")
    def hello(v: str, u: User) -> str:
        return "hi"
    pickle.loads(pickle.dumps(app))


def test_serialize_app_pydantic_type_inline():
    class User(BaseModel):
        name: str

    app = FastAPI()
    @app.get("/")
    def hello(v: str, u: User) -> str:
        return "hi"
    pickle.loads(pickle.dumps(app))

def test_serialize_app_imported():
    from mod import app
    pickle.loads(pickle.dumps(app))


def test_serialize_app_pydantic_type_closure_ref():
    class User(BaseModel):
        name: str
    def make():
        app = FastAPI()
        @app.get("/")
        def hello(v: str, u: User) -> str:
            return "hi"
        return app
    pickle.loads(pickle.dumps(make))


def test_serialize_app_pydantic_type_closure_ref_import():
    from mod import User
    def make():
        app = FastAPI()
        @app.get("/")
        def hello(v: str, u: User) -> str:
            return "hi"
        return app
    pickle.loads(pickle.dumps(make))


def test_serialize_app_pydantic_type_closure():
    def make():
        class User(BaseModel):
            name: str
        app = FastAPI()
        @app.get("/")
        def hello(v: str, u: User) -> str:
            return "hi"
        return app
    pickle.loads(pickle.dumps(make))

def test_serialize_app_imported_closure():
    from mod import closure
    pickle.loads(pickle.dumps(closure))