3D Tile editor thing

Tile3D.cs

using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

[RequireComponent(typeof(MeshFilter))]
[RequireComponent(typeof(MeshRenderer))]
[RequireComponent(typeof(MeshCollider))]
[ExecuteInEditMode]
public class Tile3D : MonoBehaviour
{
    // sides of a cube
    public static Vector3[] Sides = new Vector3[6]
    {
        Vector3.up, Vector3.down,
        Vector3.left, Vector3.right,
        Vector3.forward, Vector3.back
    };

    // individual 3d tile (used to create / manage blocks)
    // originally Faces was an array of Nullable Vector2Int but I guess Unity doesn't serialize those?
    // hence the Hidden[] array
    [Serializable]
    public class Cell
    {
        public Vector3Int Tile;
        public Vector2Int[] Faces = new Vector2Int[6];
        public bool[] Hidden = new bool[6];
    }

    // used to build the meshes
    // currently we use 2, one for what is actually being rendered and one for collisions (so the editor can click stuff)
    private class MeshBuilder
    {
        public Mesh Mesh;

        private List<Vector3> vertices = new List<Vector3>();
        private List<Vector2> uvs = new List<Vector2>();
        private List<int> triangles = new List<int>();
        private bool collider;
        private Vector2 uvTileSize;
        private float tilePadding;

        public MeshBuilder(bool collider)
        {
            Mesh = new Mesh();
            this.collider = collider;
        }

        public void Begin(Vector2 uvTileSize, float tilePadding)
        {
            this.uvTileSize = uvTileSize;
            this.tilePadding = tilePadding;

            vertices.Clear();
            uvs.Clear();
            triangles.Clear();
        }

        public void Quad(Vector3 a, Vector3 b, Vector3 c, Vector3 d, Vector2Int tile, bool hidden)
        {
            if (hidden && !collider)
                return;

            var start = vertices.Count;

            vertices.Add(a);
            vertices.Add(b);
            vertices.Add(c);
            vertices.Add(d);
            
            uvs.Add(new Vector2((tile.x + 0 + tilePadding) * uvTileSize.x, (tile.y + 0 + tilePadding) * uvTileSize.y));
            uvs.Add(new Vector2((tile.x + 1 - tilePadding) * uvTileSize.x, (tile.y + 0 + tilePadding) * uvTileSize.y));
            uvs.Add(new Vector2((tile.x + 1 - tilePadding) * uvTileSize.x, (tile.y + 1 - tilePadding) * uvTileSize.y));
            uvs.Add(new Vector2((tile.x + 0 + tilePadding) * uvTileSize.x, (tile.y + 1 - tilePadding) * uvTileSize.y));

            triangles.Add(start + 0);
            triangles.Add(start + 1);
            triangles.Add(start + 2);
            triangles.Add(start + 0);
            triangles.Add(start + 2);
            triangles.Add(start + 3);
        }

        public void End()
        {
            Mesh.Clear();
            Mesh.vertices = vertices.ToArray();
            Mesh.uv = uvs.ToArray();
            Mesh.triangles = triangles.ToArray();
            Mesh.RecalculateBounds();
            Mesh.RecalculateNormals();
        }
    }

    [HideInInspector]
    public List<Cell> Cells;
    public int TileWidth = 16;
    public int TileHeight = 16;
    public float TilePadding = 0.05f;

    public Texture Texture
    {
        get
        {
            var material = meshRenderer.sharedMaterial;
            if (material != null)
                return material.mainTexture;
            return null;
        }
    }

    private Dictionary<Vector3Int, Cell> map;

    private MeshRenderer meshRenderer;
    private MeshFilter meshFiler;
    private MeshCollider meshCollider;
    private MeshBuilder renderMeshBuilder;
    private MeshBuilder colliderMeshBuilder;

    private void OnEnable()
    {
        meshFiler = GetComponent<MeshFilter>();
        meshRenderer = GetComponent<MeshRenderer>();
        meshCollider = GetComponent<MeshCollider>();
        
        // create initial mesh
        if (renderMeshBuilder == null)
        {
            renderMeshBuilder = new MeshBuilder(false);
            colliderMeshBuilder = new MeshBuilder(true);

            meshFiler.sharedMesh = renderMeshBuilder.Mesh;
            meshCollider.sharedMesh = colliderMeshBuilder.Mesh;
        }

        // reconstruct map
        if (map == null)
        {
            map = new Dictionary<Vector3Int, Cell>();
            if (Cells != null)
                foreach (var cell in Cells)
                    map.Add(cell.Tile, cell);
        }

        // make initial cells
        if (Cells == null)
        {
            Cells = new List<Cell>();
            for (int x = -4; x < 4; x++)
                for (int z = -4; z < 4; z++)
                    Create(new Vector3Int(x, 0, z));
        }

        Rebuild();
    }
    
    public Cell Create(Vector3Int at, Vector3Int? from = null)
    {
        Cell cell;
        if (!map.TryGetValue(at, out cell))
        {
            cell = new Cell();
            cell.Tile = at;
            Cells.Add(cell);
            map.Add(at, cell);

            if (from != null)
            {
                var before = At(from.Value);
                if (before != null)
                    for (int i = 0; i < Sides.Length; i++)
                    {
                        cell.Faces[i] = before.Faces[i];
                        cell.Hidden[i] = before.Hidden[i];
                    }
            }
            else
                for (int i = 0; i < Sides.Length; i++)
                    cell.Faces[i] = new Vector2Int(0, 0);
        }

        return cell;
    }

    public void Destroy(Vector3Int at)
    {
        Cell cell;
        if (map.TryGetValue(at, out cell))
        {
            map.Remove(at);
            Cells.Remove(cell);
        }
    }

    public Cell At(Vector3Int at)
    {
        Cell cell;
        if (map.TryGetValue(at, out cell))
            return cell;
        return null;
    }

    public void Rebuild()
    {
        var uvTileSize = new Vector2(1, 1);

        // uv tile size
        var material = meshRenderer.sharedMaterial;
        if (material != null)
        {
            var texture = material.mainTexture;
            uvTileSize = new Vector2(1f / (texture.width / TileWidth), 1f / (texture.height / TileHeight));
        }

        renderMeshBuilder.Begin(uvTileSize, TilePadding);
        colliderMeshBuilder.Begin(uvTileSize, TilePadding);

        // generate each cell
        foreach (var cell in Cells)
        {
            var origin = new Vector3(cell.Tile.x + 0.5f, cell.Tile.y + 0.5f, cell.Tile.z + 0.5f);

            for (int i = 0; i < Sides.Length; i++)
            {
                var normal = new Vector3Int((int)Sides[i].x, (int)Sides[i].y, (int)Sides[i].z);
                if (At(cell.Tile + normal) == null)
                    Face(origin, Sides[i], cell.Faces[i], cell.Hidden[i]);
            }
        }

        renderMeshBuilder.End();
        colliderMeshBuilder.End();

        meshFiler.sharedMesh = renderMeshBuilder.Mesh;
        meshCollider.sharedMesh = colliderMeshBuilder.Mesh;
    }

    private void Face(Vector3 center, Vector3 normal, Vector2Int tile, bool hidden)
    {
        var up = Vector3.up;
        if (normal.y != 0)
            up = Vector2.right;

        var front = center + normal * 0.5f;
        var perp1 = Vector3.Cross(normal, up);
        var perp2 = Vector3.Cross(perp1, normal);

        var a = front + (-perp1 + perp2) * 0.5f;
        var b = front + (perp1 + perp2) * 0.5f;
        var c = front + (perp1 + -perp2) * 0.5f;
        var d = front + (-perp1 + -perp2) * 0.5f;

        renderMeshBuilder.Quad(a, b, c, d, tile, hidden);
        colliderMeshBuilder.Quad(a, b, c, d, tile, hidden);
    }

}

Tile3DEditor.cs

using System.Collections;
using System.Collections.Generic;
using UnityEditor;
using UnityEngine;

[CustomEditor(typeof(Tile3D))]
public class Tile3DEditor : Editor
{

    public Tile3D tiler { get { return (Tile3D)target; } }

    // current tool mode
    public enum ToolModes
    {
        Building,
        Painting
    }
    private ToolModes toolMode = ToolModes.Building;

    public enum PaintModes
    {
        Brush,
        Fill
    }
    private PaintModes paintMode = PaintModes.Brush;

    // used to describe a selection (tile + face)
    private class SingleSelection
    {
        public Vector3Int Tile;
        public Vector3 Face;
    }

    // used to describe multiple selections (tile(s) + face)
    private class MultiSelection
    {
        public List<Vector3Int> Tiles = new List<Vector3Int>();
        public Vector3 Face;
        public MultiSelection() { }
        public MultiSelection(SingleSelection from)
        {
            Tiles.Add(from.Tile);
            Face = from.Face;
        }
    }

    // active selections
    private SingleSelection hover = null;
    private MultiSelection selected = null;
    private Vector2Int? brush = null;
    private Vector2Int? paletteHover = null;

    public override void OnInspectorGUI()
    {
        DrawDefaultInspector();
        
        if (GUILayout.Button("Rebuild Mesh"))
            tiler.Rebuild();
    }

    protected virtual void OnSceneGUI()
    {
        var e = Event.current;
        var invokeRepaint = false;
        var draggingBlock = false;
        var interacting = (!e.control && !e.alt && e.button == 0);
        
        // override default control
        HandleUtility.AddDefaultControl(GUIUtility.GetControlID(FocusType.Passive));

        // overlay gui
        Handles.BeginGUI();
        {
            // mode toolbar
            toolMode = (ToolModes)GUI.Toolbar(new Rect(10, 10, 200, 30), (int)toolMode, new[] { "Build", "Paint" });
            if (toolMode == ToolModes.Painting)
                selected = null;

            // tileset
            if (toolMode == ToolModes.Painting)
                GUI.Window(0, new Rect(10, 70, 200, 300), PaintingWindow, "Tiles");
        }
        Handles.EndGUI();

        // Selecting & Dragging Blocks
        if (toolMode == ToolModes.Building)
        {
            // drag block in / out
            if (selected != null)
            {
                Handles.color = Color.blue;
                EditorGUI.BeginChangeCheck();

                var origin = CenterOfSelection(selected) + selected.Face * 0.5f;
                var pulled = Handles.Slider(origin, selected.Face);

                if (EditorGUI.EndChangeCheck())
                {
                    draggingBlock = true;
                    if (hover != null)
                    {
                        hover = null;
                        invokeRepaint = true;
                    }

                    // get distance and direction
                    var distance = (pulled - origin).magnitude;
                    var outwards = (int)Mathf.Sign(Vector3.Dot(pulled - origin, selected.Face));

                    // create or destroy a block (depending on direction)
                    if (distance > 1f)
                    {
                        var newTiles = new List<Vector3Int>();
                        foreach (var tile in selected.Tiles)
                        {
                            var was = tile;
                            var next = tile + selected.Face.Int() * outwards;

                            if (outwards > 0)
                                tiler.Create(next, was);
                            else
                                tiler.Destroy(was);
                            tiler.Rebuild();

                            newTiles.Add(next);
                        }

                        selected.Tiles = newTiles;
                        tiler.Rebuild();
                    }
                }
            }

            // select tiles
            if (!draggingBlock && interacting)
            {
                if (e.type == EventType.MouseDown && !e.shift)
                {
                    if (hover == null)
                        selected = null;
                    else
                        selected = new MultiSelection(hover);
                    invokeRepaint = true;
                }
                else if (e.type == EventType.MouseDrag && selected != null && hover != null && !selected.Tiles.Contains(hover.Tile))
                {
                    selected.Tiles.Add(hover.Tile);
                    invokeRepaint = true;
                }
            }
        }
        
        // active hover
        if ((e.type == EventType.MouseMove || e.type == EventType.MouseDrag) && interacting && !draggingBlock)
        {
            var next = GetSelectionAt(e.mousePosition);
            if ((hover == null && next != null) || (hover != null && next == null) || (hover != null && next != null && (hover.Tile != next.Tile || hover.Face != next.Face)))
                invokeRepaint = true;
            hover = next;
        }
        
        // painting
        if (toolMode == ToolModes.Painting && (e.type == EventType.MouseDown || e.type == EventType.MouseDrag) && interacting && hover != null)
        {
            var cell = tiler.At(hover.Tile);
            if (cell != null)
            {
                // paint single tile
                if (paintMode == PaintModes.Brush)
                {
                    if (PaintTile(cell, hover.Face, brush))
                        tiler.Rebuild();
                }
                // paint bucket
                else if (paintMode == PaintModes.Fill)
                {
                    var current = GetPaintValue(cell, hover.Face);
                    Vector3Int perp1, perp2;
                    GetPerpendiculars(hover.Face, out perp1, out perp2);

                    var active = new List<Tile3D.Cell>();
                    var filled = new HashSet<Tile3D.Cell>();
                    var directions = new Vector3Int[4] { perp1, perp1 * -1, perp2, perp2 * -1 };
                    var outwards = hover.Face.Int();

                    filled.Add(cell);
                    active.Add(cell);
                    PaintTile(cell, hover.Face, brush);

                    while (active.Count > 0)
                    {
                        var from = active[0];
                        active.RemoveAt(0);

                        for (int i = 0; i < 4; i++)
                        {
                            var next = tiler.At(from.Tile + directions[i]);
                            if (next != null && !filled.Contains(next) && tiler.At(from.Tile + directions[i] + outwards) == null && GetPaintValue(next, hover.Face) == current)
                            {
                                filled.Add(next);
                                active.Add(next);
                                PaintTile(next, hover.Face, brush);
                            }
                        }
                    }

                    tiler.Rebuild();
                }
            }
        }
        
        // Drawing
        {
            // draw hovers / selected outlines
            if (hover != null)
                DrawSelection(hover, Color.magenta);
            if (selected != null)
                DrawSelection(selected, Color.blue);

            // force repaint
            if (invokeRepaint)
                Repaint();
        }
        
        // always keep the tiler selected for now
        // later should detect if something is being grabbed or hovered
        Selection.activeGameObject = tiler.transform.gameObject;
    }

    private bool PaintTile(Tile3D.Cell cell, Vector3 face, Vector2Int? brush)
    {
        for (int i = 0; i < Tile3D.Sides.Length; i++)
        {
            if (Vector3.Dot(face, Tile3D.Sides[i]) > 0.8f)
            {
                if (brush.HasValue)
                {
                    if ((cell.Hidden[i] || cell.Faces[i] != brush.Value))
                    {
                        cell.Faces[i] = brush.Value;
                        cell.Hidden[i] = false;
                        return true;
                    }
                }
                else if (!cell.Hidden[i])
                {
                    cell.Hidden[i] = true;
                    return true;
                }
            }
        }

        return false;
    }

    private Vector2Int? GetPaintValue(Tile3D.Cell cell, Vector3 face)
    {
        for (int i = 0; i < Tile3D.Sides.Length; i++)
        {
            if (Vector3.Dot(face, Tile3D.Sides[i]) > 0.8f)
            {
                if (cell.Hidden[i])
                    return null;
                else
                    return cell.Faces[i];
            }
        }

        return null;
    }

    private Vector3 CenterOfSelection(Vector3Int tile)
    {
        return tiler.transform.position + new Vector3(tile.x + 0.5f, tile.y + 0.5f, tile.z + 0.5f);
    }

    private Vector3 CenterOfSelection(SingleSelection selection)
    {
        return CenterOfSelection(selection.Tile);
    }

    private Vector3 CenterOfSelection(MultiSelection selection)
    {
        var tile = Vector3.zero;
        foreach (var t in selection.Tiles)
            tile += new Vector3(t.x + 0.5f, t.y + 0.5f, t.z + 0.5f);
        tile /= selection.Tiles.Count;
        tile += tiler.transform.position;

        return tile;
    }

    private void DrawSelection(SingleSelection selection, Color color)
    {
        var center = CenterOfSelection(selection);
        DrawSelection(center, selection.Face, color);
    }

    private void DrawSelection(MultiSelection selection, Color color)
    {
        foreach (var tile in selection.Tiles)
            DrawSelection(CenterOfSelection(tile), selection.Face, color);
    }

    private void DrawSelection(Vector3 center, Vector3 face, Color color)
    {
        var front = center + face * 0.5f;
        Vector3 perp1, perp2;
        GetPerpendiculars(face, out perp1, out perp2);

        var a = front + (-perp1 + perp2) * 0.5f;
        var b = front + (perp1 + perp2) * 0.5f;
        var c = front + (perp1 + -perp2) * 0.5f;
        var d = front + (-perp1 + -perp2) * 0.5f;

        Handles.color = color;
        Handles.DrawDottedLine(a, b, 2f);
        Handles.DrawDottedLine(b, c, 2f);
        Handles.DrawDottedLine(c, d, 2f);
        Handles.DrawDottedLine(d, a, 2f);
    }

    private void GetPerpendiculars(Vector3 face, out Vector3 updown, out Vector3 leftright)
    {
        var up = (face.y == 0 ? Vector3.up : Vector3.right);
        updown = Vector3.Cross(face, up);
        leftright = Vector3.Cross(updown, face);
    }

    private void GetPerpendiculars(Vector3 face, out Vector3Int updown, out Vector3Int leftright)
    {
        Vector3 perp1, perp2;
        GetPerpendiculars(face, out perp1, out perp2);
        updown = perp1.Int();
        leftright = perp2.Int();
    }

    private SingleSelection GetSelectionAt(Vector2 mousePosition)
    {
        var ray = HandleUtility.GUIPointToWorldRay(mousePosition);
        var hits = Physics.RaycastAll(ray);

        foreach (var hit in hits)
        {
            var manager = hit.collider.gameObject.GetComponent<Tile3D>();
            if (manager == tiler)
            {
                var center = hit.point - hit.normal * 0.5f;

                var x = (int)Mathf.Floor(center.x - tiler.transform.position.x);
                var y = (int)Mathf.Floor(center.y - tiler.transform.position.y);
                var z = (int)Mathf.Floor(center.z - tiler.transform.position.z);

                var selection = new SingleSelection();
                selection.Tile = new Vector3Int(x, y, z);
                selection.Face = hit.normal;
                return selection;
            }
        }

        return null;
    }
    
    // This function is a giant mess but I wanted to just quickly get tile selection working
    void PaintingWindow(int id)
    {
        const int left = 10;
        const int width = 180;

        // paint mode
        paintMode = (PaintModes)GUI.Toolbar(new Rect(left, 25, width, 30), (int)paintMode, new[] { "Brush", "Fill" });

        // tiles
        if (tiler.Texture == null)
        {
            GUI.Label(new Rect(left, 64, width, 80), "Requires a Material\nwith a Texture");
        }
        else
        {
            var columns = 5;
            var top = 65;
            var tileSize = width / columns;
            var tileCount = (tiler.Texture.width / tiler.TileWidth) * (tiler.Texture.height / tiler.TileHeight) + 1;
            var tileUvSize = new Vector2(1f / (tiler.Texture.width / tiler.TileWidth), 1f / (tiler.Texture.height / tiler.TileHeight));
            var tileColumns = (tiler.Texture.width / tiler.TileWidth);
            var invokeRepaint = false;

            // get current tile we're hovering over
            var e = Event.current;
            if (e.type == EventType.MouseDown || e.type == EventType.MouseMove)
            {
                var was = paletteHover;
                paletteHover = new Vector2Int((int)((e.mousePosition.x - left) / tileSize), (int)((e.mousePosition.y - top) / tileSize));
                if (was != paletteHover)
                    invokeRepaint = true;
            }

            // note: at i=0 counts as the "empty" tile, which erases faces
            // that's why there's all the weird (i-1) stuff
            for (int i = 0; i < tileCount; i++)
            {
                var paletteX = i % columns;
                var paletteY = i / columns;
                var x = left + paletteX * tileSize;
                var y = top + paletteY * tileSize;

                // hover & select if clicked
                if (paletteHover != null && paletteHover.Value.x == paletteX && paletteHover.Value.y == paletteY)
                {
                    EditorGUI.DrawRect(new Rect(x, y, tileSize, tileSize), Color.yellow);
                    if (e.type == EventType.MouseDown && e.button == 0)
                    {
                        if (i == 0)
                            brush = null;
                        else
                            brush = new Vector2Int((i - 1) % tileColumns, (i - 1) / tileColumns);
                        invokeRepaint = true;
                        e.Use();
                    }
                }

                // draw selected
                if ((brush == null && i == 0) || (brush != null && brush.Value.x == (i - 1) % tileColumns && brush.Value.y == (i - 1) / tileColumns))
                    EditorGUI.DrawRect(new Rect(x, y, tileSize, tileSize), Color.blue);

                // draw "empty" tile
                if (i == 0)
                {
                    EditorGUI.DrawRect(new Rect(x + 2, y + 2, tileSize - 4, tileSize - 4), Color.white);
                    EditorGUI.DrawRect(new Rect(x + 2, y + 2, (tileSize - 4) / 2, (tileSize - 4) / 2), Color.gray);
                    EditorGUI.DrawRect(new Rect(x + 2 + (tileSize - 4) / 2, y + 2 + (tileSize - 4) / 2, (tileSize - 4) / 2, (tileSize - 4) / 2), Color.gray);
                }
                // draw normal tile
                else
                {
                    var tx = (i - 1) % tileColumns;
                    var ty = (i - 1) / tileColumns;
                    GUI.DrawTextureWithTexCoords(new Rect(x + 2, y + 2, tileSize - 4, tileSize - 4), tiler.Texture, new Rect(tx * tileUvSize.x, ty * tileUvSize.y, tileUvSize.x, tileUvSize.y));
                }
            }

            // force repaint
            if (invokeRepaint)
                Repaint();
        }
    }
}

public static class Tile3DUtils
{
    public static Vector3Int Int(this Vector3 vector)
    {
        return new Vector3Int((int)vector.x, (int)vector.y, (int)vector.z);
    }
}