Make media custom fields required disabling the Insert Media button while fields are empty. Example of caption and a credit custom field.

media-modal-required-fields.js

/**
 * Disables the Insert Media button while the required fields are empty.
 */
(function(){
var SelectionEventBus = _.extend({
        currentSelection: {},
        invalidSelection: {},
    }, Backbone.Events);

    var updateInvalid = function( collection ) {
        if( 'function' === typeof collection.filter ) {
            SelectionEventBus.invalidSelection = collection.filter(function (val) {
                var valid = ( val.get('caption').length > 0 && val.get('credit').length > 0 );
                val.set('valid', valid );
                return ! valid;
            });
            SelectionEventBus.trigger( 'change' );
        }
    };

    var originalView = wp.media.view.AttachmentsBrowser;
    wp.media.view.AttachmentsBrowser = originalView.extend({
        initialize: function () {
            originalView.prototype.initialize.apply(this, arguments);

            this.collection.on( 'selection:single', function( single, collection ) {
                SelectionEventBus.currentSelection = collection;
                _.delay( updateInvalid, 100, collection );
            }, this );

            this.collection.on( 'change:caption change:credit', function() {
                _.delay( updateInvalid, 100, SelectionEventBus.currentSelection );
            }, this );

            this.collection.on( 'add', _.debounce(function(){
                SelectionEventBus.trigger( 'change' );
            }, 100 ), this );

        }
    });

    var originalButtonView = wp.media.view.Button;

    wp.media.view.Button = originalButtonView.extend({
        initialize: function () {
            originalButtonView.prototype.initialize.apply(this, arguments);
            var button = this.model;
            SelectionEventBus.on('change', function(){
                button.set( 'disabled', Boolean( SelectionEventBus.invalidSelection.length ) );
            });
        },
    });
})();

media-modal-required-fields.php

<?php
/**
 * Add the credit field to the modal and media edit screen.
 */
function my_attachment_fields_to_edit($fields, $post) {
    $credit = get_post_meta_key( $post->ID, '_my_credit', true );
    $html = "<input id='attachments[$post->ID][credit]' class='credit-input' size='30' value='$credit' name='attachments[$post->ID][credit]'  />";
    $fields['credit'] = array(
        'label' => __('Credit:', 'my_domain'),
        'input' => 'html',
        'html' => $html,
        'show_in_edit' => true,
        'show_in_modal' => true,
    );

    return $fields;
}
add_filter('attachment_fields_to_edit', 'my_attachment_fields_to_edit', 10, 2);

/**
 * Make the credit available for the backbone collection.
 */
function my_prepare_attachment_for_js( $response, $attachment, $meta ) {
   $response['credit'] = get_post_meta_key( $attachment->ID, '_my_credit', true );
	 return $response;
}
add_filter('wp_prepare_attachment_for_js',  'my_prepare_attachment_for_js', 10, 3);