Insert an image into the wordpress editor always wrapped in a figure tag. The javascript is used to remove the figure element from the editor if the delete button is used. Note that the inline edit does not work with this code.

admin.js

(function($) {
  $(document).ready(function() {
    $('body').on('mousedown', '#wp_delimgbtn', function(e) {
      var editor = tinyMCE.activeEditor, 
          element = editor.selection.getNode();

      if(element.tagName !== 'FIGURE') {
        $(element).parents('figure').remove();
      }
    });
  });
}(jQuery));

optional.css

/* To remove any br or p tags that the WordPress editor likes to throw in */
figure img + br,
figure a + br,
figure img + p,
figure a + p  { display: none; }

wordpress-html5-image.php

function html5_insert_image($html, $id, $caption, $title, $align, $url, $size, $alt ) {
  //Always return an image with a <figure> tag, regardless of link or caption

  //Grab the image tag
  $image_tag = get_image_tag($id, '', $title, $align, $size);

  //Let's see if this contains a link
  $linkptrn = "/<a[^>]*>/";
  $found = preg_match($linkptrn, $html, $a_elem);

  // If no link, do nothing
  if($found > 0) {
    $a_elem = $a_elem[0];

    if(strstr($a_elem, "class=\"") !== false){ // If link already has class defined inject it to attribute
        $a_elem = str_replace("class=\"", "class=\"colorbox ", $a_elem);
    } else { // If no class defined, just add class attribute
        $a_elem = str_replace("<a ", "<a class=\"colorbox\" ", $a_elem);
    }
  } else {
    $a_elem = "";
  }

  // Set up the attributes for the caption <figure>
  $attributes  = (!empty($id) ? ' id="attachment_' . esc_attr($id) . '"' : '' );
  $attributes .= ' class="thumbnail wp-caption ' . 'align'.esc_attr($align) . '"';

  $output  = '<figure' . $attributes .'>';

  //add the image back in
  $output .= $a_elem;
  $output .= $image_tag;

  if($a_elem != "") {
    $output .= '</a>';
  }
  
  if ($caption) {
    $output .= '<figcaption class="caption wp-caption-text">'.$caption.'</figcaption>';
  }
  $output .= '</figure>';

  return $output;
}

add_filter('image_send_to_editor', 'html5_insert_image', 10, 9);
add_filter( 'disable_captions', create_function('$a', 'return true;') );

@Jozze, just in case you still need this, I came up with a solution. Here's a tinyMCE plugin that you can add to WP:

tinymce.PluginManager.add( 'advanced_image', function( editor, url ) {
  editor.on('NodeChange', function (e) { // check everytime a node is changed
    var text = editor.getBody(); // grab editor content
    var $images = $(text).find('figure'); // grab all figure elements (images)

    $images.each(function() { // loop through each found figure (image)
      if(!$('img', this).length) { // check if the figure is empty (an image was removed)
        $(this).remove(); // remove it!
      }
    });
  });
});

Hi guys,

Thanks for your hints, here's my version: it remove the figure tag when delete key, backspace key or delete image button is hit.

functions.php :

add_action('admin_init', function() {
    add_filter('mce_external_plugins', function($plugin_array ) {
        $plugin_array['figureImage'] = 'url-to-the-script/tinyMCE-figureImage.js';
        return $plugin_array;
    });
});

tinyMCE-figureImage.js :

(function() {
    tinymce.create('tinymce.plugins.figureImage', {
        init: function(editor, url) {
            const deleteHandler = function(e) {
                const editor = tinyMCE.activeEditor
                var $selection = editor.selection.getNode()
                if ($selection && $selection.tagName !== 'IMG') return

                const $figure = $selection.parentNode
                if ($figure && $figure.tagName !== 'FIGURE') return
                $figure.remove()
            }

            // Remove button
            document.querySelector('.mce-toolbar-grp div[aria-label="Remove"]').addEventListener('mousedown', deleteHandler)

            // Remove keys
            editor.on('KeyDown', function(e) {
                if ((e.keyCode == 8 || e.keyCode == 46) && editor.selection) {
                    deleteHandler()
                }
            })
        }
    })
    tinymce.PluginManager.add('figureImage', tinymce.plugins.figureImage)
})()