CreateAndDestroyComp.cmp

<aura:component >
    <lightning:button label="Create Button" onclick="{!c.createButtonDynamically}" variant="brand"/>
	  <lightning:button label="Destroy All buttons" onclick="{!c.removeButtonDynamically}" variant="destructive"/>
    <br/>
    <div style="margin:auto; height:600px; width:400px; border:2px ridge red;" aura:id="newtag">
    	{!v.body}
    </div>
</aura:component>

CreateAndDestroyCompController.js

({
	createButtonDynamically : function(component, event, helper) {
		var db=component.find("newtag");
        $A.createComponent(
        	"ui:button",
            {
                "label":"New Button"+db.get("v.body").length,
                "press": component.getReference("c.showPressedButtonLabel")
            },
            function(bn)
            {
                var bdy=db.get("v.body");
                bdy.push(bn);
                db.set("v.body",bdy);
            }
        );
        
	},
    removeButtonDynamically : function(component, event, helper){
        component.find("newtag").set("v.body",[]);
    },
    showPressedButtonLabel : function(component, event, helper){
        alert('You pressed:'+event.getSource().get("v.label"));
    }
})

gistfile1.txt

<aura:application extends="force:slds"   >
  <c:CreateAndDestroyComp />
</aura:application>