application_controller.rb

# Three things to add:
# * before_filter call
# * action_has_layout? method (if you have one, combine them)
# * adjust_for_inline
# 
class ApplicationController < ActionController::Base
  # ...
  
  before_filter :adjust_for_inline
  
  # ...
  
  private
  
  # We don't want no stinking layout if it's an AJAX request.
  def action_has_layout?
    request.format != :inline && super
  end
  
  def adjust_for_inline
    request.format = :inline if request.xhr?
  end

  # ...
end

mime_types.rb

# Add this line to your existing file.
# Jury's out on the best name for the pseudo-mimetype though.
Mime::Type.register_alias "text/html", :inline

my_controller.rb

# A long time ago, in an action far far away
def index
  # ...
  respond_to do |wants|
    wants.inline {
      # ideally use your own view (index.inline.erb)
      render :text => 'success!'
    }
  end
end