0001-Async-Chunk-Loading-and-Generation.patch

From da976f7ed15b762ef138b55b9c711184363cf364 Mon Sep 17 00:00:00 2001
From: Aikar <[email protected]>
Date: Thu, 30 Aug 2018 22:38:31 -0400
Subject: [PATCH] Async Chunk Loading and Generation

---
 .../java/net/minecraft/server/ChunkMap.java   |   3 +-
 .../minecraft/server/ChunkProviderServer.java |  24 +-
 .../minecraft/server/ChunkRegionLoader.java   |   2 +-
 .../minecraft/server/DataPaletteBlock.java    |   4 +
 .../net/minecraft/server/IChunkLoader.java    |   1 +
 .../server/PaperAsyncChunkLoader.java         | 245 ++++++++++++++++++
 .../net/minecraft/server/PlayerChunk.java     |  28 +-
 .../net/minecraft/server/SchedulerBatch.java  |   3 +
 8 files changed, 295 insertions(+), 15 deletions(-)
 create mode 100644 src/main/java/net/minecraft/server/PaperAsyncChunkLoader.java

diff --git a/src/main/java/net/minecraft/server/ChunkMap.java b/src/main/java/net/minecraft/server/ChunkMap.java
index b941676829..48cdc97377 100644
--- a/src/main/java/net/minecraft/server/ChunkMap.java
+++ b/src/main/java/net/minecraft/server/ChunkMap.java
@@ -50,6 +50,7 @@ public class ChunkMap extends Long2ObjectOpenHashMap<Chunk> {
         }
         chunk.world.timings.syncChunkLoadPostTimer.stopTiming(); // Paper
 
+        ((ChunkProviderServer) chunk.world.getChunkProvider()).asyncTaskHandler.postToMainThread(() -> {// Paper
         if (chunk.newChunk) {
             chunk.world.timings.syncChunkLoadPopulateTimer.startTiming(); // Paper
             BlockSand.instaFall = true;
@@ -73,7 +74,7 @@ public class ChunkMap extends Long2ObjectOpenHashMap<Chunk> {
             BlockSand.instaFall = false;
             chunk.world.getServer().getPluginManager().callEvent(new org.bukkit.event.world.ChunkPopulateEvent(chunk.bukkitChunk));
             chunk.world.timings.syncChunkLoadPopulateTimer.stopTiming(); // Paper
-        }
+        } }); // Paper
         // CraftBukkit end
 
         return chunk1;
diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
index c0ec896eea..620f9635fb 100644
--- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
+++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
@@ -43,15 +43,17 @@ public class ChunkProviderServer implements IChunkProvider {
     private final ChunkTaskScheduler chunkScheduler;
     private final SchedulerBatch<ChunkCoordIntPair, ChunkStatus, ProtoChunk> batchScheduler;
     public final WorldServer world;
-    private final IAsyncTaskHandler asyncTaskHandler;
+    final IAsyncTaskHandler asyncTaskHandler; // Paper
+    final PaperAsyncChunkLoader asyncChunkLoader; // Paper
 
     public ChunkProviderServer(WorldServer worldserver, IChunkLoader ichunkloader, ChunkGenerator<?> chunkgenerator, IAsyncTaskHandler iasynctaskhandler) {
         this.world = worldserver;
         this.chunkLoader = ichunkloader;
         this.chunkGenerator = chunkgenerator;
         this.asyncTaskHandler = iasynctaskhandler;
-        this.chunkScheduler = new ChunkTaskScheduler(0, worldserver, chunkgenerator, ichunkloader, iasynctaskhandler); // CraftBukkit - very buggy, broken in lots of __subtle__ ways. Same goes for async chunk loading. Also Bukkit API / plugins can't handle async events at all anyway.
+        this.chunkScheduler = new ChunkTaskScheduler(4, worldserver, chunkgenerator, ichunkloader, iasynctaskhandler); // CraftBukkit - very buggy, broken in lots of __subtle__ ways. Same goes for async chunk loading. Also Bukkit API / plugins can't handle async events at all anyway.
         this.batchScheduler = new SchedulerBatch(this.chunkScheduler);
+        this.asyncChunkLoader = new PaperAsyncChunkLoader(world, this, asyncTaskHandler, chunkLoader, this.chunkScheduler); // Paper - Async Chunks
     }
 
     public Collection<Chunk> a() {
@@ -81,7 +83,22 @@ public class ChunkProviderServer implements IChunkProvider {
     }
 
     @Nullable
-    public Chunk getChunkAt(int i, int j, boolean flag, boolean flag1) {
+    public final Chunk getChunkAt(int i, int j, boolean flag, boolean flag1) {
+        // Paper start - Async Chunks
+        return getChunkAt(i, j, flag, flag1, null);
+    }
+
+    public final Chunk getChunkAt(int x, int z, boolean load, boolean gen, Consumer<Chunk> consumer) {
+        long key = ChunkCoordIntPair.a(x, z);
+        Chunk chunk = this.chunks.get(key);
+        if (chunk != null || !load) { // return null if we aren't loading
+            return chunk;
+        }
+        return asyncChunkLoader.loadOrGenerateChunk(x, z, gen, consumer);
+    }
+    @Nullable
+    public Chunk getChunkAtOriginal(int i, int j, boolean flag, boolean flag1) { // Paper - rename to disable
+        // Paper end - Async Chunks
         IChunkLoader ichunkloader = this.chunkLoader;
         Chunk chunk;
 
@@ -171,6 +188,7 @@ public class ChunkProviderServer implements IChunkProvider {
         return this.batchScheduler.c();
     }
 
+    ReportedException generateChunkError(int i, int j, Throwable throwable) { return a(i, j, throwable); } // Paper - OBFHELPER
     private ReportedException a(int i, int j, Throwable throwable) {
         CrashReport crashreport = CrashReport.a(throwable, "Exception generating new chunk");
         CrashReportSystemDetails crashreportsystemdetails = crashreport.a("Chunk to be generated");
diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
index 06968974c5..0f216c39bd 100644
--- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java
+++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
@@ -110,7 +110,7 @@ public class ChunkRegionLoader implements IChunkLoader, IAsyncChunkSaver {
     // CraftBukkit start
     private boolean check(ChunkProviderServer cps, int x, int z) throws IOException {
         if (cps != null) {
-            com.google.common.base.Preconditions.checkState(org.bukkit.Bukkit.isPrimaryThread(), "primary thread");
+            //com.google.common.base.Preconditions.checkState(org.bukkit.Bukkit.isPrimaryThread(), "primary thread"); // Paper - this is safe
             if (cps.isLoaded(x, z)) {
                 return true;
             }
diff --git a/src/main/java/net/minecraft/server/DataPaletteBlock.java b/src/main/java/net/minecraft/server/DataPaletteBlock.java
index 5d2561a946..38e4399653 100644
--- a/src/main/java/net/minecraft/server/DataPaletteBlock.java
+++ b/src/main/java/net/minecraft/server/DataPaletteBlock.java
@@ -24,8 +24,11 @@ public class DataPaletteBlock<T> implements DataPaletteExpandable<T> {
     private int i; private int getBitsPerObject() { return this.i; } // Paper - OBFHELPER
     private final ReentrantLock j = new ReentrantLock();
 
+    Throwable locker;
     private void b() {
         if (this.j.isLocked() && !this.j.isHeldByCurrentThread()) {
+            locker.printStackTrace();
+            new Throwable("Bad writer on " + Thread.currentThread().getName()).printStackTrace();
             String s = (String) Thread.getAllStackTraces().keySet().stream().filter(Objects::nonNull).map((thread) -> {
                 return thread.getName() + ": \n\tat " + (String) Arrays.stream(thread.getStackTrace()).map(Object::toString).collect(Collectors.joining("\n\tat "));
             }).collect(Collectors.joining("\n"));
@@ -35,6 +38,7 @@ public class DataPaletteBlock<T> implements DataPaletteExpandable<T> {
             crashreportsystemdetails.a("Thread dumps", (Object) s);
             throw new ReportedException(crashreport);
         } else {
+            locker = new Throwable("locked on " + Thread.currentThread().getName());
             this.j.lock();
         }
     }
diff --git a/src/main/java/net/minecraft/server/IChunkLoader.java b/src/main/java/net/minecraft/server/IChunkLoader.java
index 4698ee99f8..33b30bb7e3 100644
--- a/src/main/java/net/minecraft/server/IChunkLoader.java
+++ b/src/main/java/net/minecraft/server/IChunkLoader.java
@@ -6,6 +6,7 @@ import javax.annotation.Nullable;
 
 public interface IChunkLoader {
 
+    default Chunk loadChunkAt(GeneratorAccess generatoraccess, int i, int j, Consumer<Chunk> consumer) throws IOException { return a(generatoraccess, i, j, consumer); } // Paper - OBFHELPER
     @Nullable
     Chunk a(GeneratorAccess generatoraccess, int i, int j, Consumer<Chunk> consumer) throws IOException;
 
diff --git a/src/main/java/net/minecraft/server/PaperAsyncChunkLoader.java b/src/main/java/net/minecraft/server/PaperAsyncChunkLoader.java
new file mode 100644
index 0000000000..04b0404aec
--- /dev/null
+++ b/src/main/java/net/minecraft/server/PaperAsyncChunkLoader.java
@@ -0,0 +1,245 @@
+/*
+ * This file is licensed under the MIT License (MIT).
+ *
+ * Copyright (c) 2018 Daniel Ennis <http://aikar.co>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package net.minecraft.server;
+
+import com.google.common.util.concurrent.MoreExecutors;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMaps;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import org.bukkit.Bukkit;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+
+public class PaperAsyncChunkLoader {
+
+    private static final ExecutorService chunkExecutor = Executors.newCachedThreadPool(new NamedIncrementingThreadFactory("Paper Chunk Executor"));
+    private static final ExecutorService directChunkExecutor = MoreExecutors.newDirectExecutorService();
+
+    private final IAsyncTaskHandler asyncTaskHandler;
+    private final ChunkProviderServer server;
+    private final WorldServer world;
+    private final IChunkLoader chunkLoader;
+    private final ChunkTaskScheduler chunkTaskScheduler;
+
+    private final Long2ObjectMap<PendingChunk> pendingChunks = Long2ObjectMaps.synchronize(new Long2ObjectOpenHashMap<>());
+
+
+    PaperAsyncChunkLoader(WorldServer world, ChunkProviderServer server, IAsyncTaskHandler asynctaskhandler, IChunkLoader chunkloader, ChunkTaskScheduler chunkTaskScheduler) {
+        this.server = server;
+        this.world = world;
+        this.asyncTaskHandler = asynctaskhandler;
+        this.chunkLoader = chunkloader;
+        this.chunkTaskScheduler = chunkTaskScheduler;
+    }
+
+    Chunk loadOrGenerateChunk(int x, int z, boolean gen, Consumer<Chunk> consumer) {
+        long key = ChunkCoordIntPair.a(x, z);
+        AtomicBoolean hadPending = new AtomicBoolean(false);
+
+        final CompletableFuture<Chunk> future = new CompletableFuture<>();
+        final PendingChunk pendingChunk = pendingChunks.compute(key, (prevKey, pending) -> {
+            if (pending == null) {
+                pending = new PendingChunk(x, z, key, gen);
+            } else {
+                hadPending.set(true);
+            }
+
+            if (!gen && pending.generating) {
+                future.complete(null);
+            } else {
+                pending.addListener(future, gen);
+            }
+            return pending;
+        });
+
+
+        if (!hadPending.get()) {
+            final boolean useDirect = !Bukkit.isPrimaryThread() || consumer == null;
+            if (useDirect) {
+                directChunkExecutor.submit(pendingChunk);
+            } else {
+                chunkExecutor.submit(pendingChunk);
+            }
+        }
+
+        if (consumer == null) {
+            try (co.aikar.timings.Timing timing = world.timings.syncChunkLoadTimer.startTiming()) {
+                return future.join();
+            }
+        } else {
+            consumeOnMain(future, consumer);
+        }
+        return null;
+    }
+
+    private Chunk loadChunk(int x, int z) {
+        try {
+            return this.chunkLoader.loadChunkAt(this.world, x, z, this::doNothingChunkConsumer);
+        } catch (IOException e) {
+            MinecraftServer.LOGGER.error("Couldn't load chunk", e);
+            return null;
+        }
+    }
+    private Chunk generateChunk(int x, int z) {
+        try (co.aikar.timings.Timing timing = world.timings.chunkGeneration.startTiming()) {
+            CompletableFuture<ProtoChunk> pending = this.chunkTaskScheduler.a(new ChunkCoordIntPair(x, z));
+
+            return new Chunk(this.world, pending.join(), x, z);
+        } catch (RuntimeException runtimeexception) {
+            throw this.server.generateChunkError(x, z, runtimeexception);
+        }
+    }
+
+    private void consumeOnMain(CompletableFuture<Chunk> future, Consumer<Chunk> consumer) {
+        future.thenAccept((c) -> this.asyncTaskHandler.postToMainThread(() -> consumer.accept(c)));
+    }
+
+    private void doNothingChunkConsumer(Chunk chunk) {}
+
+
+    static AtomicInteger idpool = new AtomicInteger(1);
+
+    public boolean chunkGoingToExists(int x, int z) {
+        synchronized (pendingChunks) {
+            long key = ChunkCoordIntPair.a(x, z);
+            PendingChunk pendingChunk = pendingChunks.get(key);
+            return pendingChunk != null && pendingChunk.canGenerate;
+        }
+    }
+
+    private class PendingChunk implements Runnable {
+        private final int x;
+        private final int z;
+        private final long key;
+
+        volatile boolean generating;
+        volatile boolean canGenerate;
+
+        final CompletableFuture<Chunk> loadOnly = new CompletableFuture<>();
+        final CompletableFuture<Chunk> generate = new CompletableFuture<>();
+        private String debug;
+
+
+
+        PendingChunk(int x, int z, long key, boolean canGenerate) {
+            this.x = x;
+            this.z = z;
+            this.key = key;
+            this.canGenerate = canGenerate;
+            debug = "(" + idpool.getAndIncrement() + ") " + x + "," + z;
+        }
+        boolean loadFinished(Chunk chunk) {
+            Chunk other = checkAddChunk(chunk);
+            boolean shouldGenerate;
+            synchronized (pendingChunks) {
+                shouldGenerate = chunk == null && other == null && canGenerate;
+                if (!shouldGenerate) {
+                    pendingChunks.remove(key);
+                } else {
+                    generating = true;
+                }
+            }
+            if (other != null) {
+                loadOnly.complete(other);
+                generate.complete(other);
+                return false;
+            } else if (chunk != null) {
+                chunk.setLastSaved(PaperAsyncChunkLoader.this.world.getTime());
+                loadOnly.complete(chunk);
+                generate.complete(chunk);
+                addEntities(chunk);
+            } else {
+                // this is nullable for anyone who doesn't need generation
+                loadOnly.complete(null);
+                if (!shouldGenerate) {
+                    generate.complete(null);
+                }
+            }
+
+            return shouldGenerate;
+        }
+
+        private void addEntities(Chunk chunk) {
+            PaperAsyncChunkLoader.this.asyncTaskHandler.postToMainThread(chunk::addEntities);
+        }
+
+        private Chunk checkAddChunk(Chunk chunk) {
+            synchronized (server.chunks) {
+                final Chunk other = server.chunks.get(key);
+                if (other == null && chunk != null) {
+                    server.chunks.put(key, chunk);
+                }
+                return other;
+            }
+        }
+
+        void generateFinished(Chunk chunk) {
+            Chunk other = checkAddChunk(chunk);
+            synchronized (pendingChunks) {
+                pendingChunks.remove(key);
+            }
+            if (other != null) {
+                chunk = other;
+            } else if (chunk != null) {
+                addEntities(chunk);
+            }
+            generate.complete(chunk);
+        }
+        void addListener(CompletableFuture<Chunk> future, boolean gen) {
+            if (gen) {
+                canGenerate = true;
+                generate.thenAccept(future::complete);
+            } else {
+                loadOnly.thenAccept(future::complete);
+            }
+        }
+
+        @Override
+        public void run() {
+            Chunk chunk = loadChunk(x, z);
+            if (!loadFinished(chunk)) {
+                return;
+            }
+
+            System.out.println("GENERATE" + debug);
+            try {
+                chunk = generateChunk(x, z);
+                System.out.println("GENERATE DONE " + debug);
+                generateFinished(chunk);
+            } catch (Exception e) {
+                e.printStackTrace();
+                System.out.println("GENERATE ERROR " + debug);
+                generateFinished(null);
+            }
+
+        }
+    }
+
+}
diff --git a/src/main/java/net/minecraft/server/PlayerChunk.java b/src/main/java/net/minecraft/server/PlayerChunk.java
index e7d465fb8a..2d39c1c64d 100644
--- a/src/main/java/net/minecraft/server/PlayerChunk.java
+++ b/src/main/java/net/minecraft/server/PlayerChunk.java
@@ -3,6 +3,7 @@ package net.minecraft.server;
 import com.google.common.collect.Lists;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import org.apache.logging.log4j.LogManager;
@@ -30,13 +31,15 @@ public class PlayerChunk {
     // All may seem good at first, but there's deeper issues if you play for a bit
     boolean chunkExists; // Paper
     private boolean loadInProgress = false;
-    private Runnable loadedRunnable = new Runnable() {
-        public void run() {
-            loadInProgress = false;
-            PlayerChunk.this.chunk = PlayerChunk.this.playerChunkMap.getWorld().getChunkProviderServer().getChunkAt(location.x, location.z, true, true);
-            markChunkUsed(); // Paper - delay chunk unloads
-        }
+    // Paper start
+    private Consumer<Chunk> chunkLoadedConsumer = chunk -> {
+        loadInProgress = false;
+        PlayerChunk pChunk = PlayerChunk.this;
+        ChunkProviderServer chunkProviderServer = pChunk.playerChunkMap.getWorld().getChunkProviderServer();
+        pChunk.chunk = chunk;// != null ? chunk : chunkProviderServer.getChunkAt(pChunk.location.x, pChunk.location.z, true, true);
+        markChunkUsed(); // Paper - delay chunk unloads
     };
+    // Paper end
     // Paper start - delay chunk unloads
     public final void markChunkUsed() {
         if (chunk != null && chunk.scheduledForUnload != null) {
@@ -52,8 +55,8 @@ public class PlayerChunk {
         ChunkProviderServer chunkproviderserver = playerchunkmap.getWorld().getChunkProviderServer();
 
         chunkproviderserver.a(i, j);
-        this.chunk = chunkproviderserver.getChunkAt(i, j, true, false);
-        this.chunkExists = this.chunk != null || ChunkIOExecutor.hasQueuedChunkLoad(playerChunkMap.getWorld(), i, j); // Paper
+        this.chunk = chunkproviderserver.getChunkAt(i, j, false, false); // Paper
+        this.chunkExists = this.chunk != null || chunkproviderserver.asyncChunkLoader.chunkGoingToExists(i, j); // Paper
         markChunkUsed(); // Paper - delay chunk unloads
     }
 
@@ -95,8 +98,13 @@ public class PlayerChunk {
         if (this.chunk != null) {
             return true;
         } else {
-            this.chunk = this.playerChunkMap.getWorld().getChunkProviderServer().getChunkAt(this.location.x, this.location.z, true, flag);
-            markChunkUsed(); // Paper - delay chunk unloads
+            // Paper start - async chunks
+            if (!loadInProgress) {
+                loadInProgress = true;
+                this.chunk = this.playerChunkMap.getWorld().getChunkProviderServer().getChunkAt(this.location.x, this.location.z, true, flag, chunkLoadedConsumer); // Paper
+                markChunkUsed(); // Paper - delay chunk unloads
+            }
+            // Paper end
             return this.chunk != null;
         }
     }
diff --git a/src/main/java/net/minecraft/server/SchedulerBatch.java b/src/main/java/net/minecraft/server/SchedulerBatch.java
index 8c88fc9c38..23965dc882 100644
--- a/src/main/java/net/minecraft/server/SchedulerBatch.java
+++ b/src/main/java/net/minecraft/server/SchedulerBatch.java
@@ -19,6 +19,7 @@ public class SchedulerBatch<K, T extends SchedulerTask<K, T>, R> {
         this.b.b();
     }
 
+    public void startBatch() { b(); } // Paper - OBFHELPER
     public void b() {
         if (this.c) {
             throw new RuntimeException("Batch already started.");
@@ -28,6 +29,7 @@ public class SchedulerBatch<K, T extends SchedulerTask<K, T>, R> {
         }
     }
 
+    public CompletableFuture<R> add(K k0) { return a(k0); } // Paper - OBFHELPER
     public CompletableFuture<R> a(K k0) {
         if (!this.c) {
             throw new RuntimeException("Batch not properly started. Please use startBatch to create a new batch.");
@@ -44,6 +46,7 @@ public class SchedulerBatch<K, T extends SchedulerTask<K, T>, R> {
         }
     }
 
+    public CompletableFuture<R> executeBatch() { return c(); } // Paper - OBFHELPER
     public CompletableFuture<R> c() {
         if (!this.c) {
             throw new RuntimeException("Batch not properly started. Please use startBatch to create a new batch.");
-- 
2.18.0