Generates patch from an in-scene camera that can replace a small part of a skybox

animated_patch_shader.gdshader

shader_type spatial;
render_mode unshaded;
// displays patch as generated by the PatchGeneratingNode


uniform sampler2DArray patch_textures:source_color, repeat_disable;

uniform float frames_per_second:hint_range(1.0, 60.0, 1.0) = 30.0;

uniform vec3 right_vector;
uniform vec3 up_vector;
uniform vec2 uv_scale = vec2(1.0, 1.0);

vec3 project_vector_to_plane(vec3 vec, vec3 normal) {
	return vec - normal * dot(vec, normal);
}

varying vec3 world_position;

void vertex()
{
    world_position = (MODEL_MATRIX * vec4(VERTEX, 1.0)).xyz;
	
	// flatten relative to the view direction
	world_position -= NODE_POSITION_WORLD;
	world_position = project_vector_to_plane(world_position, normalize(NODE_POSITION_WORLD));
	
	// offsets vertex position relative to camera position
	// the mesh will still be frustum culled based on original position, so if there's a risk consider setting custom AABB on the mesh
	VERTEX = (inverse(MODEL_MATRIX) * vec4(world_position + NODE_POSITION_WORLD + CAMERA_POSITION_WORLD, 1.0)).xyz;
}

void fragment() {
	vec2 uv2 = vec2(dot(world_position, right_vector), dot(world_position, up_vector)) / uv_scale;
	int num_frames = textureSize (patch_textures, 0).z;
	float frame = float(int(floor(TIME * frames_per_second)) % num_frames);
	vec3 uv = vec3(uv2 / 2.0 + vec2(0.5, 0.5), frame);
	
	ALBEDO.rgb = texture(patch_textures, uv).rgb;
}

generate_patch.gd

@tool
class_name GeneratePatch extends EditorScript

# helps sets up the patch generating node, adding any selected meshes to the list of patches

const image_path:String = "res://%s.png" # %s will be replaced with the name of any selected MeshInstance3Ds

const image_size:int = 512

var patch_shader:Shader = load("res://generate_patch/patch_shader.gdshader")

var animated_patch_shader:Shader = load("res://generate_patch/animated_patch_shader.gdshader")

# TODO does this actually matter if it's a sky patch?
const camera_position:Vector3 = Vector3.ZERO

var patch_gen:PatchGeneratingNode

var camera:Camera3D


func _run() -> void:
	var selections = EditorInterface.get_selection().get_selected_nodes()
	print("node count = ", selections.size())
	
	if selections.size() == 0: # no point
		print ("nothing in scene is selected")
		return
	
	# collect a list of selected meshes as target meshes to render to
	var patch_meshes:Array[MeshInstance3D] = []
	
	for selection in selections:
		if selection is MeshInstance3D:
			patch_meshes.append(selection as MeshInstance3D)
	
	# instantiate viewport
	patch_gen = PatchGeneratingNode.new()
	patch_gen.name = "patch_generator"
	patch_gen.size = Vector2i(image_size, image_size)
	patch_gen.image_size = Vector2i(image_size, image_size)
	patch_gen.patch_shader = patch_shader
	patch_gen.animated_patch_shader = animated_patch_shader
	get_scene().add_child(patch_gen)
	patch_gen.owner = get_scene()
	
	# instantiate camera
	camera = Camera3D.new()
	patch_gen.add_child(camera)
	patch_gen.camera = camera
	camera.name = "patch_camera"
	camera.global_position = camera_position
	camera.owner = get_scene()
	
	patch_gen.patch_meshes = patch_meshes
	
	EditorInterface.edit_node(patch_gen) #need viewport selected or it won't render
	
	patch_gen.run_now = true
	# the remainder is handled in the PatchGeneratingNode

patch_generating_node.gd

@tool
extends SubViewport
class_name PatchGeneratingNode

# if you have something that should update for each frame of an animation, listen to this
signal image_rendered()

# list of meshes in-scene that will have patches assigned to them
@export var patch_meshes:Array[MeshInstance3D]

# camera used to render patches
@export var camera:Camera3D

# size of the image we're saving
@export var image_size:Vector2i

# how much we downsample that image
@export var down_sample:int = 2

# shader to use for patch shader
@export var patch_shader:Shader

# shader used for animated patch
@export var animated_patch_shader:Shader

# if rendering multiple frames
@export var target_number_of_frames:int = 1

# where to save individual images
# %s will be replaced with name of patch MeshInstance3D and %d will be replaced with frame number
@export var image_path_name:String = "res://%s_%02d.png"

# where it will save a texture array
@export var texture_array_path_name:String = "res://%s_texture_array.png"

# starts the process running
@export var run_now:bool = false:
	set(new_value):
		if new_value:
			run_capture()
		run_now = false
	get():
		return run_now

func run_capture() -> void:
	print ("running 01")
	
	# hide patches so they don't show up in patches
	for patch in patch_meshes:
		patch.visible = false
		
	size = image_size * down_sample
	await get_tree().process_frame # let the size change take hold
	
	for patch in patch_meshes:
		print ("running 02")
		camera.look_at(patch.global_position - camera.global_position)
		
		var max_offset_x:float = 0.0 # max distance bounds are offset this direction
		var max_offset_y:float = 0.0
		var patch_bounds:AABB = patch.mesh.get_aabb()
		
		var forward_vector:Vector3 = (patch.global_position - camera.global_position).normalized()
		
		var right_vector:Vector3 = forward_vector.cross(Vector3.UP).normalized()
		var up_vector:Vector3 = forward_vector.cross(right_vector).normalized()
		
		
		for i in range(0, 8):
			var point:Vector3 = patch_bounds.get_endpoint(i)
			point = patch.transform * point # get position in world space
			var relative_point:Vector3 = (point - patch.global_position) # world space position relative to patch
			max_offset_x = max(max_offset_x, abs(relative_point.dot(right_vector)))
			max_offset_y = max(max_offset_y, abs(relative_point.dot(up_vector)))
			point = point - camera.global_position # get position relative to camera

		
		var max_offset:float = max(max_offset_x, max_offset_y)
		var h_degrees = rad_to_deg(forward_vector.angle_to(patch.global_position - camera.global_position + right_vector * max_offset_x))
		var v_degrees = rad_to_deg(forward_vector.angle_to(patch.global_position - camera.global_position + up_vector * max_offset_y))
		
		# TODO make this account for variable aspect ratios
		camera.fov = max(h_degrees, v_degrees) * 2.0
		
		# check if current resolution is too high or low for this fov
		var assumed_fov:float = 70.0 # what we assume is a normal FOV player is viewing this sky at
		var assumed_resolution:float = 3840.0 # max resolution
		# compare that to actual image size
		var calc_size:float = assumed_resolution / assumed_fov * camera.fov
		if image_size.x < calc_size / 2.0:
			printerr("image size should be bigger ", calc_size)
		elif image_size.x > calc_size * 2.0:
			printerr("image size should be smaller ", calc_size)
		
		# set up shader and material on patch
		var patch_display_material = ShaderMaterial.new()
		if (target_number_of_frames==1):
			patch_display_material.shader = patch_shader
		else:
			patch_display_material.shader = animated_patch_shader
		
		patch_display_material.set_shader_parameter("right_vector", right_vector)
		patch_display_material.set_shader_parameter("up_vector", up_vector)
		patch_display_material.set_shader_parameter("uv_scale", Vector2(max_offset, max_offset))
		
		var image_array:Array[Image]
		for i in range(0, target_number_of_frames):
			await get_tree().process_frame # wait for camera to render
			
			var rendered_texture = get_texture()
			
			var got_image = rendered_texture.get_image()
			
			# TODO add this to an array of textures
			var image_file_name:String = image_path_name%[patch.name, i]
			got_image.resize(image_size.x, image_size.y, Image.INTERPOLATE_LANCZOS)
			
			got_image.save_png(image_file_name)
			
			await get_tree().process_frame
			
			var texture_image = ImageTexture.create_from_image(got_image)
			
			#EditorInterface.get_resource_filesystem().scan() # get it to process just saved image 
			#await get_tree().process_frame
			#var loaded_image:Image = Image.load_from_file(image_file_name)
			#texture_image.create_from_image(loaded_image) # TODO how to make it use the loaded image instead?
			
			if target_number_of_frames>1:
				image_rendered.emit() # anything that wants to update now that we've rendered
			else:
				patch_display_material.set_shader_parameter("patch_texture", texture_image)
				
			image_array.append(got_image)
			
		
		print("right_vector = ", right_vector, " - ", right_vector.length())
		print("up_vector = ", up_vector, " - ", up_vector.length())
		print("scale = ", max_offset)
		
		if (target_number_of_frames>1):
			save_animated_patch(image_array, texture_array_path_name%[patch.name])
			# TODO how to assign the generated texturearray to the material?
			# patch_display_material.set_shader_parameter("patch_texture_array", texture_array)
		
		patch.set_surface_override_material(0, patch_display_material)
		
	for patch in patch_meshes:
		patch.visible = true

# finds the factors of an integer
static func find_factors(integer:int) -> Array[int]:
	var factors:Array[int]
	for i in range(1, integer + 1):
		if integer / i * i == integer:
			factors.append(i)
	return factors

# finds two factors closest to forming a square
static func find_square_factors(integer:int) -> Vector2i:
	var factors = find_factors(integer)
	# find factors closest to square root
	var root:float = sqrt(float(integer))
	var min_diff:float = integer
	var y:int = 1
	for factor in factors:
		var diff:float = abs(factor - root)
		if diff < min_diff:
			min_diff = diff
			y = factor
	
	var x:int = integer / y
	return Vector2i(x, y)

func save_animated_patch(image_array: Array[Image], path_name:String) -> void:
	if image_array == null || image_array.size() == 0:
		return # don't bother
	var image_count:int = image_array.size()
	var img_sq:Vector2i = find_square_factors(image_count)
	var image_size:Vector2i = Vector2i(image_array[0].get_width(), image_array[0].get_height())
	var new_image:Image = Image.create(image_size.x * img_sq.x, image_size.y * img_sq.y, false, image_array[0].get_format())
	# copy source images to target image
	var index:int = 0
	var source_rect:Rect2i = Rect2i(0, 0, image_size.x, image_size.y)
	for y in range(0, img_sq.y):
		for x in range(0, img_sq.x):
			var position:Vector2i = Vector2i(x * image_size.x, y * image_size.y)
			new_image.blit_rect(image_array[index], source_rect, position)
			index += 1
	new_image.save_png(path_name)
	
	# set up import settings on the image
	await get_tree().process_frame
	EditorInterface.get_resource_filesystem().scan() # 
	await get_tree().process_frame
	
	var import_settings:ConfigFile = ConfigFile.new()
	import_settings.load(path_name + ".import")
	print(import_settings)
	import_settings.set_value("remap", "importer", "2d_array_texture")
	import_settings.set_value("params", "slices/horizontal", img_sq.x)
	import_settings.set_value("params", "slices/vertical", img_sq.y)
	import_settings.set_value("params", "mipmaps/generate", false)
	import_settings.save(path_name + ".import")
	
	EditorInterface.get_resource_filesystem().scan() 

patch_shader.gdshader

shader_type spatial;
render_mode unshaded;
// displays patch as generated by the PatchGeneratingNode


uniform sampler2D patch_texture:source_color;

uniform vec3 right_vector;
uniform vec3 up_vector;
uniform vec2 uv_scale = vec2(1.0, 1.0);

vec3 project_vector_to_plane(vec3 vec, vec3 normal) {
	return vec - normal * dot(vec, normal);
}

varying vec3 world_position;

void vertex()
{
    world_position = (MODEL_MATRIX * vec4(VERTEX, 1.0)).xyz;
	
	// flatten relative to the view direction
	world_position -= NODE_POSITION_WORLD;
	world_position = project_vector_to_plane(world_position, normalize(NODE_POSITION_WORLD));
	
	// offsets vertex position relative to camera position
	// the mesh will still be frustum culled based on original position, so if there's a risk consider setting custom AABB on the mesh
	VERTEX = (inverse(MODEL_MATRIX) * vec4(world_position + NODE_POSITION_WORLD + CAMERA_POSITION_WORLD, 1.0)).xyz;
}

void fragment() {
	vec2 uv2 = vec2(dot(world_position, right_vector), dot(world_position, up_vector)) / uv_scale;
	vec2 uv = uv2 / 2.0 + vec2(0.5, 0.5);
	ALBEDO.rgb = texture(patch_texture, uv).rgb;
}