Blender: Prototype method for applying decimation modifiers with shape keys

apply_top_modifier_with_shape_keys_decimation_aware.py

# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# For more information, please refer to <http://unlicense.org>

# -- Prototype method for applying decimation modifiers with shape keys --
# This works just fine with Planar and probably with Un-Subdivide but won't work with Collapse.
# This is because Collapse actually alters the geometry, making mapping it essentially impossible.

bl_info = {
	"name": "ApplyDecimationModifierWithShapeKeys",
	"blender": (2, 80, 0),
	"category": "Object"
}

import bpy
import bmesh

# A polished script would account for naming conflicts here (or at least make it less collidable).
LAYER_NAME = "ReliableVertexID"

def set_selection(context, active, selected):
	for obj in context.scene.objects:
		obj.select_set(False)
	for obj in selected:
		obj.select_set(True)
	bpy.context.view_layer.objects.active = active

def step1_2_associate_vertex_ids(context, obj):
	# Step 1: Add ReliableVertexID attribute.
	set_selection(context, obj, [obj])
	bpy.ops.object.mode_set(mode = "OBJECT")
	obj.data.attributes.new(LAYER_NAME, "INT", "POINT")
	# Step 2: Generate reliable vertex IDs. We will use this to cross-associate vertex positions later.
	bpy.ops.object.mode_set(mode = "EDIT")
	dat = bmesh.from_edit_mesh(obj.data)
	vertex_id_layer = dat.verts.layers.int[LAYER_NAME]
	for vtx in dat.verts:
		vtx[vertex_id_layer] = vtx.index + 1
	bmesh.update_edit_mesh(obj.data, loop_triangles = False, destructive = False)
	bpy.ops.object.mode_set(mode = "OBJECT")

def step3_transfer_modifier_to_proxy(context, obj):
	# Step 3: For various reasons, we can't really properly trace a modifier that *both* changes topology *and* positions without a lot of effort.
	# (Basically, if there's a vertex in Basis's output that isn't in the modifier output for a non-Basis key, you can only leave it unchanged, which is breaky.)
	# But the thing is, *that's not how Decimate works.* It only changes topology.
	set_selection(context, obj, [obj])
	bpy.ops.object.duplicate()
	new_proxy = context.active_object
	set_selection(context, obj, [obj])
	bpy.ops.object.modifier_remove(modifier = obj.modifiers[0].name)
	set_selection(context, new_proxy, [new_proxy])
	bpy.ops.object.shape_key_remove(all = True, apply_mix = True)
	# Apply the modifier; the proxy now has the new topology with the cross-association data.
	bpy.ops.object.modifier_apply(modifier = new_proxy.modifiers[0].name)
	return new_proxy

def get_copyable_layers(layers_from, layers_to):
	result = []
	# copy_from isn't enough to transfer layer data, so we have to work something out.
	for kind in ["bool", "color", "deform", "float", "float_color", "float_vector", "freestyle", "int", "skin", "string", "uv"]:
		try:
			lc_from = layers_from.__getattribute__(kind)
			lc_to = layers_to.__getattribute__(kind)
		except Exception as ex:
			print("Exception in get_copyable_layers (kind, " + kind + "): " + str(ex))
			continue
		for k, layer_from in lc_from.items():
			# print("matcher", k, layer_from)
			try:
				layer_to = lc_to[k]
				result.append((layer_from, layer_to))
			except Exception as ex:
				print("Exception in get_copyable_layers (match, " + kind + "): " + str(ex))
				pass
	# print("matcher", str(result))
	return result

def do_copy_layers(layers_copy, elm_from, elm_to):
	for layer_from, layer_to in layers_copy:
		# print("copy " + str(layer_from) + ", " + str(layer_to) + ", " + str(elm_to))
		elm_to[layer_to] = elm_from[layer_from]

def step4_copy_topology(context, obj_from, obj_to):
	# Step 4: This is what the setup was for.
	bm_from = bmesh.new()
	bm_from.from_mesh(obj_from.data)
	# rebuild lookup tables, from_mesh leaves them out of date
	bm_from.faces.ensure_lookup_table()
	bm_from.edges.ensure_lookup_table()
	bm_from.verts.ensure_lookup_table()
	# continue
	set_selection(context, obj_to, [obj_to])
	bpy.ops.object.mode_set(mode = "EDIT")
	bm_to = bmesh.from_edit_mesh(obj_to.data)
	# Step 4.1: Delete all faces and edges in bm_to.
	bm_to.faces.ensure_lookup_table()
	for face in bm_to.faces:
		bm_to.faces.remove(face)
	bm_to.edges.ensure_lookup_table()
	for edge in bm_to.edges:
		bm_to.edges.remove(edge)
	bm_to.edges.ensure_lookup_table()
	bm_to.faces.ensure_lookup_table()
	bm_to.verts.ensure_lookup_table()
	# -- mapper --
	bm_from_idlayer = bm_from.verts.layers.int[LAYER_NAME]
	def map_vtxs(vtxs):
		total = []
		for vtx in vtxs:
			true_id = vtx[bm_from_idlayer] - 1
			if true_id < 0 or true_id >= len(bm_to.verts):
				return None
			total.append(bm_to.verts[true_id])
		return total
	# Step 4.2: Recreate edges in bm_to and copy settings.
	edges_copyable = get_copyable_layers(bm_from.edges.layers, bm_to.edges.layers)
	for edge_from in bm_from.edges:
		edge_mapped = map_vtxs(edge_from.verts)
		if edge_mapped != None:
			edge_to = bm_to.edges.new(edge_mapped)
			if not edge_to.is_valid:
				continue
			# print("edge found to be valid")
			# copy intrinsic attributes
			edge_to.seam = edge_from.seam
			edge_to.smooth = edge_from.smooth
			# continue...
			do_copy_layers(edges_copyable, edge_from, edge_to)
	# Step 4.3: Recreate faces in bm_to and copy settings (and loops, and layers on both).
	faces_copyable = get_copyable_layers(bm_from.faces.layers, bm_to.faces.layers)
	loops_copyable = get_copyable_layers(bm_from.loops.layers, bm_to.loops.layers)
	for face_from in bm_from.faces:
		face_mapped = map_vtxs(face_from.verts)
		if face_mapped != None:
			face_to = bm_to.faces.new(face_mapped)
			if not face_to.is_valid:
				continue
			# print("face found to be valid", len(face_from.loops), len(face_to.loops))
			# copy intrinsic attributes
			face_to.material_index = face_from.material_index
			face_to.normal = face_from.normal
			face_to.smooth = face_from.smooth
			# continue...
			do_copy_layers(faces_copyable, face_from, face_to)
			for i in range(len(face_from.loops)):
				loop_from = face_from.loops[i]
				loop_to = face_to.loops[i]
				do_copy_layers(loops_copyable, loop_from, loop_to)
	# Finally, update edit mesh.
	bmesh.update_edit_mesh(obj_to.data)
	bpy.ops.object.mode_set(mode = "OBJECT")

def step5_delete_attribute(context, obj):
	# Step 5: Delete ReliableVertexID attribute.
	set_selection(context, obj, [obj])
	bpy.ops.object.mode_set(mode = "OBJECT")
	obj.data.attributes.remove(obj.data.attributes[LAYER_NAME])

class ApplyDecimationModifierWithShapeKeys(bpy.types.Operator):
	bl_idname = "object.apply_decimation_modifier_with_shape_keys"
	bl_label = "Apply Decimation Modifier With Shape Keys"
	bl_options = {'REGISTER', 'UNDO'}

	def execute(self, context):

		# This doesn't check for anything like possible linked meshes or anything like that.
		# We also don't do any 'general cleanup' i.e. mode control, active object, etc.
		# We assume a single selected & active object and a well-behaved state.
		# This script shows the method only.

		obj = context.active_object

		step1_2_associate_vertex_ids(context, obj)
		proxy_obj = step3_transfer_modifier_to_proxy(context, obj)
		step4_copy_topology(context, proxy_obj, obj)
		bpy.data.objects.remove(proxy_obj)
		step5_delete_attribute(context, obj)

		return {'FINISHED'}

def menu_func(self, context):
	self.layout.operator(ApplyDecimationModifierWithShapeKeys.bl_idname)

def register():
	bpy.utils.register_class(ApplyDecimationModifierWithShapeKeys)
	bpy.types.VIEW3D_MT_object.append(menu_func)

def unregister():
	bpy.utils.unregister_class(ApplyDecimationModifierWithShapeKeys)
	bpy.types.VIEW3D_MT_object.remove(menu_func)

if __name__ == "__main__":
	register()