Initial support for loading "proto big cube" converted 1.12.2 CC worlds

build.gradle

@@ -237,10 +237,11 @@ dependencies {
     }
 
     // modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
-//
-    modRuntime ("supercoder79:databreaker:0.2.8") {
-        exclude module: "fabric-loader"
-    }
+
+    // This is commented out as it (obviously) breaks loading old worlds, and should probably be enabled in dev env
+//    modRuntime ("supercoder79:databreaker:0.2.8") {
+//        exclude module: "fabric-loader"
+//    }
 
     compileOnly('com.google.code.findbugs:jsr305:3.0.1')
     testCompileOnly('com.google.code.findbugs:jsr305:3.0.1')

src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/MixinChunkMap.java

@@ -240,7 +240,7 @@ private void onConstruct(ServerLevel serverLevel, LevelStorageSource.LevelStorag
             this.cubeQueueSorter.createExecutor(lightMailbox, false));
 
         try {
-            regionCubeIO = new RegionCubeIO(file, "chunk", "cube");
+            regionCubeIO = new RegionCubeIO(file, dataFixer, "chunk", "cube");
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }

src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinEntityStorage.java

@@ -49,7 +49,7 @@ public abstract class MixinEntityStorage implements CubicEntityStorage {
     @Inject(method = "<init>", at = @At("RETURN"))
     private void setupCubeIO(ServerLevel serverLevel, File file, DataFixer dataFixer, boolean bl, Executor executor, CallbackInfo ci) throws IOException {
         if (((CubicLevelHeightAccessor) serverLevel).isCubic()) {
-            cubeWorker = new RegionCubeIO(file, file.getName(), file.getName());
+            cubeWorker = new RegionCubeIO(file, dataFixer, file.getName(), file.getName());
         }
     }
 

src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinSectionStorage.java

@@ -81,7 +81,7 @@ private void getServerLevel(File file, Function<Runnable, Codec<R>> function, Fu
                                 LevelHeightAccessor heightAccessor, CallbackInfo ci) throws IOException {
 
         if (((CubicLevelHeightAccessor) levelHeightAccessor).isCubic()) {
-            cubeWorker = new RegionCubeIO(file, file.getName() + "-chunk", file.getName());
+            cubeWorker = new RegionCubeIO(file, dataFixer, file.getName() + "-chunk", file.getName());
         }
     }
 

src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java

@@ -2,10 +2,13 @@
 
 import static net.minecraft.nbt.NbtIo.writeCompressed;
 
+import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.Map;
@@ -18,23 +21,37 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.zip.GZIPInputStream;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.mojang.datafixers.DataFixer;
 import com.mojang.datafixers.util.Either;
 import cubicchunks.regionlib.impl.EntryLocation2D;
 import cubicchunks.regionlib.impl.EntryLocation3D;
 import cubicchunks.regionlib.impl.SaveCubeColumns;
 import io.github.opencubicchunks.cubicchunks.CubicChunks;
+import io.github.opencubicchunks.cubicchunks.utils.Coords;
 import io.github.opencubicchunks.cubicchunks.world.level.CubePos;
+import net.minecraft.SharedConstants;
 import net.minecraft.nbt.CompoundTag;
+import net.minecraft.nbt.ListTag;
+import net.minecraft.nbt.NbtAccounter;
 import net.minecraft.nbt.NbtIo;
+import net.minecraft.nbt.NbtUtils;
+import net.minecraft.nbt.Tag;
+import net.minecraft.resources.ResourceKey;
+import net.minecraft.util.datafix.DataFixTypes;
 import net.minecraft.util.thread.ProcessorMailbox;
 import net.minecraft.util.thread.StrictQueue;
 import net.minecraft.world.level.ChunkPos;
+import net.minecraft.world.level.Level;
+import net.minecraft.world.level.chunk.ChunkStatus;
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
+import net.minecraft.world.level.storage.DimensionDataStorage;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 
@@ -51,16 +68,18 @@ public class RegionCubeIO {
     private final Map<ChunkPos, SaveEntry> pendingChunkWrites = Maps.newLinkedHashMap();
     private final Map<CubePos, SaveEntry> pendingCubeWrites = Maps.newLinkedHashMap();
 
+    private final DataFixer dataFixer;
+
     private final ProcessorMailbox<StrictQueue.IntRunnable> chunkExecutor;
     private final ProcessorMailbox<StrictQueue.IntRunnable> cubeExecutor;
 
     private final AtomicBoolean shutdownRequested = new AtomicBoolean();
 
     private final Executor executor = Executors.newSingleThreadExecutor(IO_WORKER_FACTORY);
 
-    public RegionCubeIO(File storageFolder, @Nullable String chunkWorkerName, String cubeWorkerName) throws IOException {
+    public RegionCubeIO(File storageFolder, DataFixer dataFixer, @Nullable String chunkWorkerName, String cubeWorkerName) throws IOException {
         this.storageFolder = storageFolder;
-
+        this.dataFixer = dataFixer;
 
         this.chunkExecutor = new ProcessorMailbox<>(new StrictQueue.FixedPriorityQueue(Priority.values().length), executor, "RegionCubeIO-" + chunkWorkerName);
         this.cubeExecutor = new ProcessorMailbox<>(new StrictQueue.FixedPriorityQueue(Priority.values().length), executor, "RegionCubeIO-" + cubeWorkerName);
@@ -142,6 +161,22 @@ public CompletableFuture<Void> saveCubeNBT(CubePos cubePos, CompoundTag cubeNBT)
                         return Either.left(null);
                     }
 
+                    boolean isOldData = false;
+                    try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(buf.get().array()))) {
+                        // NBT can't begin with byte 255, so this acts as a marker for
+                        // 1.12.2 "proto-big-cube" converted data
+                        if (in.read() == 255) {
+                            // a second byte with value 0 for potential future extension
+                            if (in.read() != 0) {
+                                throw new RuntimeException("Invalid data, expected 0");
+                            }
+                            isOldData = true;
+                        }
+                    }
+                    if (isOldData) {
+                        return Either.left(loadOldNbt(cubePos, buf.get().array()));
+                    }
+
                     CompoundTag compoundnbt = NbtIo.readCompressed(new ByteArrayInputStream(buf.get().array()));
                     return Either.left(compoundnbt);
                 } catch (Exception exception) {
@@ -308,6 +343,126 @@ private void storePendingChunk() {
         }
     }
 
+    @Nullable
+    private CompoundTag loadOldNbt(CubePos pos, byte[] data) throws IOException {
+        ByteArrayInputStream in = new ByteArrayInputStream(data);
+        CompoundTag[] cubeTags = new CompoundTag[8];
+        try (InputStream gzip = new BufferedInputStream(new GZIPInputStream(in))) {
+            // skip byte 255 and byte 0
+            gzip.read();
+            gzip.read();
+            for (int i = 0; i < 8; i++) {
+                cubeTags[i] = NbtIo.read(new DataInputStream(gzip), NbtAccounter.UNLIMITED);
+            }
+        }
+
+        return mergeOldNbt(pos, cubeTags);
+    }
+
+    @Nullable
+    private CompoundTag mergeOldNbt(CubePos pos, CompoundTag[] cubeTags) {
+        CompoundTag outTag = new CompoundTag();
+        for (int i = 0; i < cubeTags.length; i++) {
+            CompoundTag level = cubeTags[i].getCompound("Level");
+            level.put("TerrainPopulated", level.get("populated"));
+            level.put("LightPopulated", level.get("initLightDone"));
+            ListTag tileTicks = level.getList("TileTicks", CompoundTag.TAG_COMPOUND);
+
+            // prepare tile ticks to that high bits of "y" coordinate are actually cube section index
+            for (Tag tileTick : tileTicks) {
+                CompoundTag tick = (CompoundTag) tileTick;
+                int x = tick.getInt("x");
+                int y = tick.getInt("y");
+                int z = tick.getInt("z");
+                int idx = Coords.blockToIndex(x, y, z);
+                tick.putInt("y", y & 0xF | idx << 4);
+            }
+
+            int version = ChunkStorage.getVersion(cubeTags[i]);
+
+            try {
+                cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], version, 1493);
+                //if (nbt.getCompound("Level").getBoolean("hasLegacyStructureData")) {
+                //    if (this.legacyStructureHandler == null) {
+                //        this.legacyStructureHandler = LegacyStructureDataHandler.getLegacyStructureHandler(worldKey, (DimensionDataStorage)persistentStateManagerFactory.get());
+                //    }
+                //    nbt = this.legacyStructureHandler.updateFromLegacy(nbt);
+                //}
+                cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], Math.max(1493, version));
+            } catch (Exception e) {
+                System.err.printf("Vanilla DFU error on (%d, %d, %d)!\n", pos.getX(), pos.getY(), pos.getZ());
+                e.printStackTrace(System.err);
+            }
+            if (cubeTags[i] == null) {
+                LOGGER.warn("Dropping incomplete cube at " + pos);
+                return null;
+            }
+        }
+        CompoundTag levelOut = new CompoundTag();
+        levelOut.putInt("xPos", pos.getX());
+        levelOut.putInt("yPos", pos.getY());
+        levelOut.putInt("zPos", pos.getZ());
+
+        outTag.put("ToBeTicked", new ListTag());
+        outTag.put("Level", levelOut);
+        // TODO: biomes
+        outTag.putIntArray("Biomes", new int[8 * 8 * 8]);
+        for (int i = 0; i < cubeTags.length; i++) {
+            CompoundTag cube = cubeTags[i];
+            CompoundTag level = cube.getCompound("Level");
+            ListTag toBeTicked = cube.getList("ToBeTicked", Tag.TAG_LIST);
+            ListTag outToBeTicked = outTag.getList("ToBeTicked", Tag.TAG_LIST);
+            for (int i1 = 0; i1 < toBeTicked.size(); i1++) {
+                ListTag toTickEntry = toBeTicked.getList(i1);
+                ListTag toTickEntryOut = outToBeTicked.getList(i1);
+                toTickEntryOut.addAll(toTickEntry);
+                outToBeTicked.set(i1, toTickEntryOut);
+            }
+
+            level.putLong("LastUpdate", Math.max(level.getLong("LastUpdate"), levelOut.getLong("LastUpdate")));
+            level.putLong("InhabitedTime", Math.max(level.getLong("InhabitedTime"), levelOut.getLong("InhabitedTime")));
+
+            ChunkStatus status = ChunkStatus.byName(level.getString("Status"));
+            ChunkStatus oldStatus = levelOut.contains("Status") ? ChunkStatus.byName(level.getString("Status")) : null;
+            ChunkStatus newStatus = oldStatus == null ? status : oldStatus.isOrAfter(ChunkStatus.SPAWN) ? status : ChunkStatus.EMPTY;
+            levelOut.putString("Status", newStatus.getName());
+
+            ListTag sections = levelOut.getList("Sections", Tag.TAG_COMPOUND);
+            levelOut.put("Sections", sections);
+            CompoundTag section = level.getList("Sections", Tag.TAG_COMPOUND).getCompound(0);
+            section.putShort("i", (short) i);
+            sections.add(section);
+
+            levelOut.putBoolean("isLightOn", true);
+
+            ListTag tileEntities = levelOut.getList("TileEntities", Tag.TAG_COMPOUND);
+            levelOut.put("TileEntities", tileEntities);
+            ListTag tileEntitiesOld = level.getList("TileEntities", Tag.TAG_COMPOUND);
+            tileEntities.addAll(tileEntitiesOld);
+
+            ListTag entities = levelOut.getList("Entities", Tag.TAG_COMPOUND);
+            levelOut.put("Entities", entities);
+            ListTag entitiesOld = level.getList("Entities", Tag.TAG_COMPOUND);
+            entities.addAll(entitiesOld);
+
+        }
+        return outTag;
+    }
+
+    public CompoundTag upgradeChunkTag(ResourceKey<Level> worldKey, Supplier<DimensionDataStorage> persistentStateManagerFactory, CompoundTag nbt) {
+        int i = ChunkStorage.getVersion(nbt);
+        if (i < 1493) {
+            throw new IllegalArgumentException("Pre-1.17 version handled elsewhere, but trying " + i);
+        }
+
+        nbt = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, nbt, Math.max(1493, i));
+        if (i < SharedConstants.getCurrentVersion().getWorldVersion()) {
+            nbt.putInt("DataVersion", SharedConstants.getCurrentVersion().getWorldVersion());
+        }
+
+        return nbt;
+    }
+
     static class SaveEntry {
         private CompoundTag data;
         private final CompletableFuture<Void> result = new CompletableFuture<>();