Python check yaml with a model (pydantic vs schema vs marshmallow)

Python - yaml & model (pydantic, schema, marshmallow)

Some example about validation of yaml

With one of them :

- pydantic
  - Error of validation : easy readable
  - Only defined fields covered

- marshmallow
  - Error of validation : easy readable
  - Must be exhaustive (or use 'partial')

- schema
  - error on garden, None not accepted (not solved but useless yet because too verbose)

yaml_marshmallow.py

from marshmallow import Schema, fields
import yaml


yaml_home = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name: "bike"
        wheels: 2
        engine: "feet"
"""

# underground not defined
yaml_home_notdefined = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    underground:
        nothing:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name: "bike"
        wheels: 2
        engine: "feet"
"""

# wrong value for vehicle
yaml_home_wrongvalue = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name:
          wheels: 2
        engine: "feet"
    pouet: ""

"""

class SchemaVehicle(Schema):
    name = fields.Str()
    wheels = fields.Int()
    engine = fields.Str()

class SchemaNode(Schema):
    vehicle = fields.Nested(SchemaVehicle)

class SchemaGarden(SchemaNode):
    pass
    
class SchemaOutside(SchemaNode):
    garden = fields.Nested(SchemaGarden,allow_none=True)

class SchemaGarage(SchemaNode):
    pass
    
class SchemaInside(SchemaNode):
    garage = fields.Nested(SchemaGarage)

class SchemaRoot(SchemaNode):
    outside = fields.Nested(SchemaOutside)
    inside = fields.Nested(SchemaInside)
    

########## OUTPUT ##########
print(10 *'-', "good", 10 *'-')
dict_home = yaml.safe_load(yaml_home)
result = SchemaRoot().load(dict_home)
pprint(result)

>>> '''
---------- good ----------
{'inside': {'garage': {'vehicle': {'engine': 'hands',
                                   'name': 'trash can',
                                   'wheels': 2}},
            'vehicle': {'engine': 'feet', 'name': 'bike', 'wheels': 2}},
 'outside': {'garden': None,
             'vehicle': {'engine': 'hands', 'name': 'trash can', 'wheels': 2}}}
'''
########## OUTPUT ##########
print(10 *'-', "underground not defined", 10 *'-')
dict_home_notdefined = yaml.safe_load(yaml_home_notdefined)
result2 = SchemaRoot().load(dict_home_notdefined)
pprint(result2)

>>> '''
---------- underground not defined ----------

---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
<ipython-input-104-14f54681f398> in <module>
      1 print(10 *'-', "underground not defined", 10 *'-')
      2 dict_home_notdefined = yaml.safe_load(yaml_home_notdefined)
----> 3 result2 = SchemaRoot().load(dict_home_notdefined)
      4 pprint(result2)

~/.local/lib/python3.6/site-packages/marshmallow/schema.py in load(self, data, many, partial, unknown)
    713         """
    714         return self._do_load(
--> 715             data, many=many, partial=partial, unknown=unknown, postprocess=True
    716         )
    717 

~/.local/lib/python3.6/site-packages/marshmallow/schema.py in _do_load(self, data, many, partial, unknown, postprocess)
    894             exc = ValidationError(errors, data=data, valid_data=result)
    895             self.handle_error(exc, data, many=many, partial=partial)
--> 896             raise exc
    897 
    898         return result

ValidationError: {'inside': {'underground': ['Unknown field.']}}
'''

########## OUTPUT ##########

print(10 *'-', "wrong value for vehicle", 10 *'-')
dict_home_wrongvalue = yaml.safe_load(yaml_home_wrongvalue)
result3 = SchemaRoot().load(dict_home_wrongvalue)
pprint(result3)

>>>'''
---------- wrong value for vehicle ----------

---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
<ipython-input-106-0111fee7c5ce> in <module>
      1 print(10 *'-', "wrong value for vehicle", 10 *'-')
      2 dict_home_wrongvalue = yaml.safe_load(yaml_home_wrongvalue)
----> 3 result3 = SchemaRoot().load(dict_home_wrongvalue)
      4 pprint(result3)

~/.local/lib/python3.6/site-packages/marshmallow/schema.py in load(self, data, many, partial, unknown)
    713         """
    714         return self._do_load(
--> 715             data, many=many, partial=partial, unknown=unknown, postprocess=True
    716         )
    717 

~/.local/lib/python3.6/site-packages/marshmallow/schema.py in _do_load(self, data, many, partial, unknown, postprocess)
    894             exc = ValidationError(errors, data=data, valid_data=result)
    895             self.handle_error(exc, data, many=many, partial=partial)
--> 896             raise exc
    897 
    898         return result

ValidationError: {'inside': {'vehicle': {'name': ['Not a valid string.']}, 'pouet': ['Unknown field.']}}
'''

yaml_pydantic.py

import yaml
import json
from pprint import pprint
from pydantic import BaseModel, ValidationError, parse_obj_as, validator


yaml_home = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name: "bike"
        wheels: 2
        engine: "feet"
"""

# underground not defined
yaml_home_notdefined = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    underground:
        nothing:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name: "bike"
        wheels: 2
        engine: "feet"
"""

# wrong value for vehicle
yaml_home_wrongvalue = """---
outside:
    vehicle:
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    garage:
        vehicle:
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle:
        name:
          wheels: 2
        engine: "feet"
    pouet: ""

"""


class ModelVehicle(BaseModel):
    name: str
    wheels: int
    engine: str

class ModelNode(BaseModel):
    vehicle: ModelVehicle = None

class ModelOutside(ModelNode):
    garden: str = None

class ModelGarage(ModelNode):
    pass
    
class ModelInside(ModelNode):
    garage: ModelGarage = None

class ModelRoot(ModelNode):
    outside: ModelOutside
    inside: ModelInside

        
########## OUTPUT ##########
print(10 *'-', "good", 10 *'-')
dict_home = yaml.safe_load(yaml_home)
result = ModelRoot(**dict_home)
pprint(json.loads(result.json()))

>>>'''
---------- good ----------
{'inside': {'garage': {'vehicle': {'engine': 'hands',
                                   'name': 'trash can',
                                   'wheels': 2}},
            'vehicle': {'engine': 'feet', 'name': 'bike', 'wheels': 2}},
 'outside': {'garden': None,
             'vehicle': {'engine': 'hands', 'name': 'trash can', 'wheels': 2}},
 'vehicle': None}
'''

########## OUTPUT ##########
print(10 *'-', "underground not defined", 10 *'-')
dict_home_notdefined = yaml.safe_load(yaml_home_notdefined)
result2 = ModelRoot(**dict_home_notdefined)
pprint(json.loads(result2.json()))

>>>'''
---------- underground not defined ----------
{'inside': {'garage': {'vehicle': {'engine': 'hands',
                                   'name': 'trash can',
                                   'wheels': 2}},
            'vehicle': {'engine': 'feet', 'name': 'bike', 'wheels': 2}},
 'outside': {'garden': None,
             'vehicle': {'engine': 'hands', 'name': 'trash can', 'wheels': 2}},
 'vehicle': None}
'''

########## OUTPUT ##########
print(10 *'-', "wrong value for vehicle", 10 *'-')
dict_home_wrongvalue = yaml.safe_load(yaml_home_wrongvalue)
result3 = ModelRoot(**dict_home_wrongvalue)
pprint(json.loads(result3.json()))

>>> '''---------- wrong value for vehicle ----------

---------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
<ipython-input-142-9702e34ba967> in <module>
      2 dict_home_wrongvalue = yaml.safe_load(yaml_home_wrongvalue)
      3 # result3 = parse_obj_as(ModelRoot, yaml_home_wrongvalue)
----> 4 result3 = ModelRoot(**dict_home_wrongvalue)
      5 pprint(json.loads(result3.json()))

~/.local/lib/python3.6/site-packages/pydantic/main.cpython-36m-x86_64-linux-gnu.so in pydantic.main.BaseModel.__init__()

ValidationError: 2 validation errors for ModelRoot
inside -> vehicle -> name
  str type expected (type=type_error.str)
inside -> vehicle -> wheels
  field required (type=value_error.missing)

'''

yaml_schema.py

from schema import Optional, Schema, SchemaError
import yaml

yaml_home = """---
outside:
    vehicle :
      name: "trash can"
      wheels: 2
      engine: "hands"
    garden:
inside:
    garage:
        vehicle :
            name: "trash can"
            wheels: 2
            engine: "hands"
    vehicle :
        name: "bike"
        wheels: 2
        engine: "feet"
"""

dict_home = yaml.safe_load(yaml_home)

schema_vehicle = {
    'name': str,
    'wheels': int,
    'engine': str
}

schema_node = {
    Optional('vehicle'): schema_vehicle
}

schema_garden = schema_node.copy()

schema_outside = {
    Optional('garden'): schema_garden
}
schema_outside.update(schema_node)

schema_garage = schema_node.copy()

schema_intside = {
    'garage': schema_garden
}
schema_intside.update(schema_node)

schema_root = {
    'outside': schema_outside,
    'inside': schema_intside
}
schema_root.update(schema_node)

result = Schema(schema_root).validate(dict_home)

pprint(result)

####

SchemaUnexpectedTypeError                 Traceback (most recent call last)
~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    395                                 try:
--> 396                                     nvalue = Schema(svalue, error=e, ignore_extra_keys=i).validate(value)
    397                                 except SchemaError as x:

~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    359             exitstack = ExitStack()
--> 360             data = Schema(dict, error=e).validate(data)
    361             new = type(data)()  # new - is a dict of the validated values

~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    430                 message = self._prepend_schema_name(message)
--> 431                 raise SchemaUnexpectedTypeError(message, e.format(data) if e else None)
    432         if flavor == VALIDATOR:

SchemaUnexpectedTypeError: None should be instance of 'dict'

During handling of the above exception, another exception occurred:

SchemaError                               Traceback (most recent call last)
~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    395                                 try:
--> 396                                     nvalue = Schema(svalue, error=e, ignore_extra_keys=i).validate(value)
    397                                 except SchemaError as x:

~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    399                                     message = self._prepend_schema_name(k)
--> 400                                     raise SchemaError([message] + x.autos, [e.format(data) if e else None] + x.errors)
    401                                 else:

SchemaError: Key 'garden' error:
None should be instance of 'dict'

During handling of the above exception, another exception occurred:

SchemaError                               Traceback (most recent call last)
<ipython-input-18-a828f22d8616> in <module>
     53 schema_root.update(schema_node)
     54 
---> 55 result = Schema(schema_root).validate(dict_home)
     56 
     57 pprint(result)

~/.local/lib/python3.6/site-packages/schema.py in validate(self, data)
    398                                     k = "Key '%s' error:" % nkey
    399                                     message = self._prepend_schema_name(k)
--> 400                                     raise SchemaError([message] + x.autos, [e.format(data) if e else None] + x.errors)
    401                                 else:
    402                                     new[nkey] = nvalue

SchemaError: Key 'outside' error:
Key 'garden' error:
None should be instance of 'dict'