description of unexpected behaviour with copy_to and nested_types and dynamic_templates

test_copy_to_and_dynamic_templates_with_nested_documents.sh

curl -XDELETE "http://localhost:9200/test"
curl -XPUT "http://localhost:9200/test/"

# create strict mapping of nested in nested type with copy_to
curl -XPUT "http://localhost:9200/test/searchdoc_fix/_mapping" -d'
{
  "searchdoc_fix": {
    "dynamic": "strict",
    "properties": {
      "du": {
        "dynamic": "strict",
        "type": "nested",
        "include_in_root": "true",
        "properties": {
          "_all": {
            "type": "string"
          },
          "du_type": {
            "type": "string"
          },
          "titleabstract": {
            "type": "string"
          },
          "du_title": {
	    "dynamic": "strict",
            "type": "nested",
            "include_in_root": "true",
            "properties": {
              "_all": {
                "type": "string"
              },
              "title_type": {
                "type": "string"
              },
              "du_title4shti": {
                "type": "object",
                "properties": {
                  "_all": {
                    "type": "string"
                  },
                  "title": {
                    "type": "string",
                    "copy_to": [
                      "du.du_title.du_title4shti._all",
                      "du.du_title._all",
                      "du._all",
                      "du.titleabstract"
                    ]
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}'

curl -XPUT "http://localhost:9200/test/searchdoc_fix/1" -d'
{
  "du": [
    {
      "du_type": "Sendung",
      "du_title": [
        {
          "title_type": "Sende-/Haupttitel",
          "du_title4shti": {
            "title": "Merkel trifft Kohl"
          }
        }
      ]
    }
  ]
}'
#insert OK


#now test if all properties are copied with object notation to the root-layer

# root layer & no nesting: root _all field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "match": {
         "_all": "Kohl"
      }
   }
}'
#hit found - OK

# root layer & no nesting: first nesting level _all field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "match": {
         "du._all": "Kohl"
      }
   }
}'
#hit found - OK

# root layer & no nesting: second nesting level _all field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "match": {
         "du.du_title._all": "Kohl"
      }
   }
}'
#hit found - OK

# root layer & no nesting: _all field in object of second nesting level works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "match": {
         "du.du_title.du_title4shti._all": "Kohl"
      }
   }
}'
#hit found - OK

# root layer & no nesting: first nesting level custom field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "match": {
         "du.titleabstract": "Kohl"
      }
   }
}'
#hit found - OK

#now test if properties are copied with object notation to the second nested layer

#second nesting layer: object _all field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du.du_title",
         "query": {
            "match": {
               "du.du_title.du_title4shti._all": "Kohl"
            }
         }
      }
   }
}'
#hit found - OK

#second nesting layer: _all field works
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du.du_title",
         "query": {
            "match": {
               "du.du_title._all": "Kohl"
            }
         }
      }
   }
}'
#hit found - OK

#second nesting layer: ERROR: Why do I get field of the first nesting layer?
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du.du_title",
         "query": {
            "match": {
               "du._all": "Kohl"
            }
         }
      }
   }
}'
#hit found - unexpected! WHY?

#second nesting layer: ERROR: Why do I get field of the first nesting layer?
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du.du_title",
         "query": {
            "match": {
               "du.titleabstract": "Kohl"
            }
         }
      }
   }
}'
#hit found - unexpected! WHY?


#second nesting layer: ERROR: Why do I get field of the first nesting layer?
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du",
         "query": {
            "match": {
               "du.titleabstract": "Kohl"
            }
         }
      }
   }
}'
# no hit found - unexpected! WHY?

#now test if properties are copied with object notation to the first nested layer

#first nesting layer: ERROR: Why do I NOT get field copied to the first nesting layer?
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du",
         "query": {
            "match": {
               "du._all": "Kohl"
            }
         }
      }
   }
}'
# no hit found - unexpected! WHY?


#first nesting layer: I don´t get fields from object in the second nested layer. OK
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du",
         "query": {
            "match": {
               "du.du_title.du_title4shti._all": "Kohl"
            }
         }
      }
   }
}'
# no hit found - OK


#first nesting layer: I don´t get fields from the second nested layer. OK
curl -XGET "http://localhost:9200/test/searchdoc_fix/_search" -d'
{
   "query": {
      "nested": {
         "path": "du",
         "query": {
            "match": {
               "du.du_title._all": "Kohl"
            }
         }
      }
   }
}'
# no hit found - OK



#now make things worse: test dynamic template mapping with copy_to and multi layered nested fields

#define mapping with dynamic template and nested types
curl -XPUT "http://localhost:9200/test/searchdoc_dynamic/_mapping" -d'
{
  "searchdoc_dynamic": {
    "dynamic": "strict",
    "dynamic_templates": [
      {
        "nested_du_title": {
          "match": "du_title4*",
          "mapping": {
            "type": "object",
            "properties": {
              "_all": {
                "type": "string"
              },
              "titleabstract": {
                "type": "string"
              },
              "title": {
                "type": "string",
                "copy_to": [
                  "du.du_title.{name}._all",
                  "du.du_title._all",
                  "du.titleabstract"
                ]
              }
            }
          }
        }
      }
    ],
    "properties": {
      "du": {
        "dynamic": "strict",
        "type": "nested",
        "include_in_root": "true",
        "properties": {
          "_all": {
            "type": "string"
          },
          "du_type": {
            "type": "string"
          },
          "du_title": {
            "type": "nested",
            "include_in_root": "true",
            "dynamic": "true",
            "properties": {
              "_all": {
                "type": "string"
              },
              "title_type": {
                "type": "string"
              }
            }
          }
        }
      }
    }
  }
}'


# now it is´nt possible to index a document because of non-existing object. why? How can I fix it? 
curl -XPUT "http://localhost:9200/test/searchdoc_dynamic/1" -d'
{
  "du": [
    {
      "du_type": "Sendung",
      "du_title": [
        {
          "title_type": "Sende-/Haupttitel",
          "du_title4shti": {
            "title": "Merkel trifft Kohl"
          }
        }
      ]
    }
  ]
}'
#{
#   "error": "MapperParsingException[attempt to copy value to non-existing object [du.du_title.du_title4shti._all]]",
#   "status": 400
#}
# WHY?



#the dynamic mapping should be used for documents like this:
curl -XPUT "http://localhost:9200/test/searchdoc/2" -d'
{
  "du": [
    {
      "du_type": "Sendung",
      "du_title": [
        {
          "title_type": "Sende-/Haupttitel",
          "du_title4shti": [
            {
              "title": "Kohl wieder Kanzler"
            }
          ]
        }
      ]
    },
    {
      "du_type": "Beitrag",
      "du_title": [
        {
          "title_type": "Beitragstitel",
          "du_title4beti": [
            {
              "title": "Merkel tritt zurück"
            }
          ]
        },
        {
          "title_type": "Arbeitstitel",
          "du_title4arti": [
            {
              "title": "Kohl verabschiedet Merkel"
            }
          ]
        }
        
      ]
    }
  ]
}'