Insert an image into the wordpress editor always wrapped in a figure tag. The javascript is used to remove the figure element from the editor if the delete button is used. Note that the inline edit does not work with this code.

admin.js

(function($) {
  $(document).ready(function() {
    $('body').on('mousedown', '#wp_delimgbtn', function(e) {
      var editor = tinyMCE.activeEditor, 
          element = editor.selection.getNode();

      if(element.tagName !== 'FIGURE') {
        $(element).parents('figure').remove();
      }
    });
  });
}(jQuery));

optional.css

/* To remove any br or p tags that the WordPress editor likes to throw in */
figure img + br,
figure a + br,
figure img + p,
figure a + p  { display: none; }

wordpress-html5-image.php

function html5_insert_image($html, $id, $caption, $title, $align, $url, $size, $alt ) {
  //Always return an image with a <figure> tag, regardless of link or caption

  //Grab the image tag
  $image_tag = get_image_tag($id, '', $title, $align, $size);

  //Let's see if this contains a link
  $linkptrn = "/<a[^>]*>/";
  $found = preg_match($linkptrn, $html, $a_elem);

  // If no link, do nothing
  if($found > 0) {
    $a_elem = $a_elem[0];

    if(strstr($a_elem, "class=\"") !== false){ // If link already has class defined inject it to attribute
        $a_elem = str_replace("class=\"", "class=\"colorbox ", $a_elem);
    } else { // If no class defined, just add class attribute
        $a_elem = str_replace("<a ", "<a class=\"colorbox\" ", $a_elem);
    }
  } else {
    $a_elem = "";
  }

  // Set up the attributes for the caption <figure>
  $attributes  = (!empty($id) ? ' id="attachment_' . esc_attr($id) . '"' : '' );
  $attributes .= ' class="thumbnail wp-caption ' . 'align'.esc_attr($align) . '"';

  $output  = '<figure' . $attributes .'>';

  //add the image back in
  $output .= $a_elem;
  $output .= $image_tag;

  if($a_elem != "") {
    $output .= '</a>';
  }
  
  if ($caption) {
    $output .= '<figcaption class="caption wp-caption-text">'.$caption.'</figcaption>';
  }
  $output .= '</figure>';

  return $output;
}

add_filter('image_send_to_editor', 'html5_insert_image', 10, 9);
add_filter( 'disable_captions', create_function('$a', 'return true;') );

just change '#wp_delimgbtn' to 'div[aria-label="Remove"]'

(function($) {
    $(document).ready(function() {
         $('body').on('mousedown', 'div[aria-label="Remove"]' , function(e) {
            var editor = tinyMCE.activeEditor,
                element = editor.selection.getNode();

            if(element.tagName !== 'FIGURE') {
                $(element).parents('figure').remove();
            }
        });
    });
}(jQuery));

Sorry, kind of new to Wordpress. Where does the admin.js code go? In the functions php file? Or as an external script in the header.php file?
Where does the wordpress-html5-image.php file go?

Don't understand why WP is not supporting figure elements out of the box.

@roman-soerensen
I used your 'div[aria-label="Remove"]'-variant, but everytime i use the delete button, i get a js error and the editor gets cleared

js error: Uncaught TypeError: Cannot read property 'nodeName' of null

I suspect that tinymce wants to delete the tag, but can't after I already deleted the parent with the click event... did this kind of thing happen to any of you?

I enqueued the admin script with jquery as a dependency.

@Jozze, just in case you still need this, I came up with a solution. Here's a tinyMCE plugin that you can add to WP:

tinymce.PluginManager.add( 'advanced_image', function( editor, url ) {
  editor.on('NodeChange', function (e) { // check everytime a node is changed
    var text = editor.getBody(); // grab editor content
    var $images = $(text).find('figure'); // grab all figure elements (images)

    $images.each(function() { // loop through each found figure (image)
      if(!$('img', this).length) { // check if the figure is empty (an image was removed)
        $(this).remove(); // remove it!
      }
    });
  });
});

Hi guys,

Thanks for your hints, here's my version: it remove the figure tag when delete key, backspace key or delete image button is hit.

functions.php :

add_action('admin_init', function() {
    add_filter('mce_external_plugins', function($plugin_array ) {
        $plugin_array['figureImage'] = 'url-to-the-script/tinyMCE-figureImage.js';
        return $plugin_array;
    });
});

tinyMCE-figureImage.js :

(function() {
    tinymce.create('tinymce.plugins.figureImage', {
        init: function(editor, url) {
            const deleteHandler = function(e) {
                const editor = tinyMCE.activeEditor
                var $selection = editor.selection.getNode()
                if ($selection && $selection.tagName !== 'IMG') return

                const $figure = $selection.parentNode
                if ($figure && $figure.tagName !== 'FIGURE') return
                $figure.remove()
            }

            // Remove button
            document.querySelector('.mce-toolbar-grp div[aria-label="Remove"]').addEventListener('mousedown', deleteHandler)

            // Remove keys
            editor.on('KeyDown', function(e) {
                if ((e.keyCode == 8 || e.keyCode == 46) && editor.selection) {
                    deleteHandler()
                }
            })
        }
    })
    tinymce.PluginManager.add('figureImage', tinymce.plugins.figureImage)
})()