From b6d0e16fa10fd34eefe5d20442845b344138835d Mon Sep 17 00:00:00 2001 From: Bartosz Skrzypczak Date: Mon, 28 Mar 2022 01:01:19 +0100 Subject: [PATCH 1/3] Initial support for loading "proto big cube" converted 1.12.2 CC worlds --- build.gradle | 3 - .../core/common/chunk/MixinChunkMap.java | 2 +- .../chunk/storage/MixinEntityStorage.java | 2 +- .../chunk/storage/MixinSectionStorage.java | 2 +- .../world/storage/RegionCubeIO.java | 154 +++++++++++++++++- 5 files changed, 155 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index dca0ebe3a..4bbade273 100644 --- a/build.gradle +++ b/build.gradle @@ -238,9 +238,6 @@ dependencies { // modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" // - modRuntime ("supercoder79:databreaker:0.2.8") { - exclude module: "fabric-loader" - } compileOnly('com.google.code.findbugs:jsr305:3.0.1') testCompileOnly('com.google.code.findbugs:jsr305:3.0.1') diff --git a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/MixinChunkMap.java b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/MixinChunkMap.java index e47360e28..f043bbf06 100644 --- a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/MixinChunkMap.java +++ b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/MixinChunkMap.java @@ -240,7 +240,7 @@ private void onConstruct(ServerLevel serverLevel, LevelStorageSource.LevelStorag this.cubeQueueSorter.createExecutor(lightMailbox, false)); try { - regionCubeIO = new RegionCubeIO(file, "chunk", "cube"); + regionCubeIO = new RegionCubeIO(file, dataFixer, "chunk", "cube"); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinEntityStorage.java b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinEntityStorage.java index 6bffa9197..b06cc59db 100644 --- a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinEntityStorage.java +++ b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinEntityStorage.java @@ -49,7 +49,7 @@ public abstract class MixinEntityStorage implements CubicEntityStorage { @Inject(method = "", at = @At("RETURN")) private void setupCubeIO(ServerLevel serverLevel, File file, DataFixer dataFixer, boolean bl, Executor executor, CallbackInfo ci) throws IOException { if (((CubicLevelHeightAccessor) serverLevel).isCubic()) { - cubeWorker = new RegionCubeIO(file, file.getName(), file.getName()); + cubeWorker = new RegionCubeIO(file, dataFixer, file.getName(), file.getName()); } } diff --git a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinSectionStorage.java b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinSectionStorage.java index 8769a7788..bba62546a 100644 --- a/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinSectionStorage.java +++ b/src/main/java/io/github/opencubicchunks/cubicchunks/mixin/core/common/chunk/storage/MixinSectionStorage.java @@ -81,7 +81,7 @@ private void getServerLevel(File file, Function> function, Fu LevelHeightAccessor heightAccessor, CallbackInfo ci) throws IOException { if (((CubicLevelHeightAccessor) levelHeightAccessor).isCubic()) { - cubeWorker = new RegionCubeIO(file, file.getName() + "-chunk", file.getName()); + cubeWorker = new RegionCubeIO(file, dataFixer, file.getName() + "-chunk", file.getName()); } } diff --git a/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java b/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java index 6ed558963..e66a9a283 100644 --- a/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java +++ b/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java @@ -2,10 +2,13 @@ import static net.minecraft.nbt.NbtIo.writeCompressed; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Map; @@ -18,23 +21,37 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Supplier; +import java.util.zip.GZIPInputStream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.mojang.datafixers.DataFixer; import com.mojang.datafixers.util.Either; import cubicchunks.regionlib.impl.EntryLocation2D; import cubicchunks.regionlib.impl.EntryLocation3D; import cubicchunks.regionlib.impl.SaveCubeColumns; import io.github.opencubicchunks.cubicchunks.CubicChunks; +import io.github.opencubicchunks.cubicchunks.utils.Coords; import io.github.opencubicchunks.cubicchunks.world.level.CubePos; +import net.minecraft.SharedConstants; import net.minecraft.nbt.CompoundTag; +import net.minecraft.nbt.ListTag; +import net.minecraft.nbt.NbtAccounter; import net.minecraft.nbt.NbtIo; +import net.minecraft.nbt.NbtUtils; +import net.minecraft.nbt.Tag; +import net.minecraft.resources.ResourceKey; +import net.minecraft.util.datafix.DataFixTypes; import net.minecraft.util.thread.ProcessorMailbox; import net.minecraft.util.thread.StrictQueue; import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.Level; +import net.minecraft.world.level.chunk.ChunkStatus; +import net.minecraft.world.level.chunk.storage.ChunkStorage; +import net.minecraft.world.level.storage.DimensionDataStorage; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -51,6 +68,8 @@ public class RegionCubeIO { private final Map pendingChunkWrites = Maps.newLinkedHashMap(); private final Map pendingCubeWrites = Maps.newLinkedHashMap(); + private final DataFixer dataFixer; + private final ProcessorMailbox chunkExecutor; private final ProcessorMailbox cubeExecutor; @@ -58,9 +77,9 @@ public class RegionCubeIO { private final Executor executor = Executors.newSingleThreadExecutor(IO_WORKER_FACTORY); - public RegionCubeIO(File storageFolder, @Nullable String chunkWorkerName, String cubeWorkerName) throws IOException { + public RegionCubeIO(File storageFolder, DataFixer dataFixer, @Nullable String chunkWorkerName, String cubeWorkerName) throws IOException { this.storageFolder = storageFolder; - + this.dataFixer = dataFixer; this.chunkExecutor = new ProcessorMailbox<>(new StrictQueue.FixedPriorityQueue(Priority.values().length), executor, "RegionCubeIO-" + chunkWorkerName); this.cubeExecutor = new ProcessorMailbox<>(new StrictQueue.FixedPriorityQueue(Priority.values().length), executor, "RegionCubeIO-" + cubeWorkerName); @@ -142,6 +161,22 @@ public CompletableFuture saveCubeNBT(CubePos cubePos, CompoundTag cubeNBT) return Either.left(null); } + boolean isOldData = false; + try (GZIPInputStream in = new GZIPInputStream(new ByteArrayInputStream(buf.get().array()))) { + // NBT can't begin with byte 255, so this acts as a marker for + // 1.12.2 "proto-big-cube" converted data + if (in.read() == 255) { + // a second byte with value 0 for potential future extension + if (in.read() != 0) { + throw new RuntimeException("Invalid data, expected 0"); + } + isOldData = true; + } + } + if (isOldData) { + return Either.left(loadOldNbt(cubePos, buf.get().array())); + } + CompoundTag compoundnbt = NbtIo.readCompressed(new ByteArrayInputStream(buf.get().array())); return Either.left(compoundnbt); } catch (Exception exception) { @@ -308,6 +343,121 @@ private void storePendingChunk() { } } + @Nullable + private CompoundTag loadOldNbt(CubePos pos, byte[] data) throws IOException { + ByteArrayInputStream in = new ByteArrayInputStream(data); + CompoundTag[] cubeTags = new CompoundTag[8]; + try (InputStream gzip = new BufferedInputStream(new GZIPInputStream(in))) { + // skip byte 255 and byte 0 + gzip.read(); + gzip.read(); + for (int i = 0; i < 8; i++) { + cubeTags[i] = NbtIo.read(new DataInputStream(gzip), NbtAccounter.UNLIMITED); + } + } + + return mergeOldNbt(pos, cubeTags); + } + + @Nullable + private CompoundTag mergeOldNbt(CubePos pos, CompoundTag[] cubeTags) { + CompoundTag outTag = new CompoundTag(); + for (int i = 0; i < cubeTags.length; i++) { + CompoundTag level = cubeTags[i].getCompound("Level"); + level.put("TerrainPopulated", level.get("populated")); + level.put("LightPopulated", level.get("initLightDone")); + ListTag tileTicks = level.getList("TileTicks", CompoundTag.TAG_COMPOUND); + + // prepare tile ticks to that high bits of "y" coordinate are actually cube section index + for (Tag tileTick : tileTicks) { + CompoundTag tick = (CompoundTag) tileTick; + int x = tick.getInt("x"); + int y = tick.getInt("y"); + int z = tick.getInt("z"); + int idx = Coords.blockToIndex(x, y, z); + tick.putInt("y", y & 0xF | idx << 4); + } + + int version = ChunkStorage.getVersion(cubeTags[i]); + + cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], version, 1493); + //if (nbt.getCompound("Level").getBoolean("hasLegacyStructureData")) { + // if (this.legacyStructureHandler == null) { + // this.legacyStructureHandler = LegacyStructureDataHandler.getLegacyStructureHandler(worldKey, (DimensionDataStorage)persistentStateManagerFactory.get()); + // } + // nbt = this.legacyStructureHandler.updateFromLegacy(nbt); + //} + cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], Math.max(1493, version)); + if (cubeTags[i] == null) { + LOGGER.warn("Dropping incomplete cube at " + pos); + return null; + } + } + CompoundTag levelOut = new CompoundTag(); + levelOut.putInt("xPos", pos.getX()); + levelOut.putInt("yPos", pos.getY()); + levelOut.putInt("zPos", pos.getZ()); + + outTag.put("ToBeTicked", new ListTag()); + outTag.put("Level", levelOut); + // TODO: biomes + outTag.putIntArray("Biomes", new int[8 * 8 * 8]); + for (int i = 0; i < cubeTags.length; i++) { + CompoundTag cube = cubeTags[i]; + CompoundTag level = cube.getCompound("Level"); + ListTag toBeTicked = cube.getList("ToBeTicked", Tag.TAG_LIST); + ListTag outToBeTicked = outTag.getList("ToBeTicked", Tag.TAG_LIST); + for (int i1 = 0; i1 < toBeTicked.size(); i1++) { + ListTag toTickEntry = toBeTicked.getList(i1); + ListTag toTickEntryOut = outToBeTicked.getList(i1); + toTickEntryOut.addAll(toTickEntry); + outToBeTicked.set(i1, toTickEntryOut); + } + + level.putLong("LastUpdate", Math.max(level.getLong("LastUpdate"), levelOut.getLong("LastUpdate"))); + level.putLong("InhabitedTime", Math.max(level.getLong("InhabitedTime"), levelOut.getLong("InhabitedTime"))); + + ChunkStatus status = ChunkStatus.byName(level.getString("Status")); + ChunkStatus oldStatus = levelOut.contains("Status") ? ChunkStatus.byName(level.getString("Status")) : null; + ChunkStatus newStatus = oldStatus == null ? status : oldStatus.isOrAfter(ChunkStatus.SPAWN) ? status : ChunkStatus.EMPTY; + levelOut.putString("Status", newStatus.getName()); + + ListTag sections = levelOut.getList("Sections", Tag.TAG_COMPOUND); + levelOut.put("Sections", sections); + CompoundTag section = level.getList("Sections", Tag.TAG_COMPOUND).getCompound(0); + section.putShort("i", (short) i); + sections.add(section); + + levelOut.putBoolean("isLightOn", true); + + ListTag tileEntities = levelOut.getList("TileEntities", Tag.TAG_COMPOUND); + levelOut.put("TileEntities", tileEntities); + ListTag tileEntitiesOld = level.getList("TileEntities", Tag.TAG_COMPOUND); + tileEntities.addAll(tileEntitiesOld); + + ListTag entities = levelOut.getList("Entities", Tag.TAG_COMPOUND); + levelOut.put("Entities", entities); + ListTag entitiesOld = level.getList("Entities", Tag.TAG_COMPOUND); + entities.addAll(entitiesOld); + + } + return outTag; + } + + public CompoundTag upgradeChunkTag(ResourceKey worldKey, Supplier persistentStateManagerFactory, CompoundTag nbt) { + int i = ChunkStorage.getVersion(nbt); + if (i < 1493) { + throw new IllegalArgumentException("Pre-1.17 version handled elsewhere, but trying " + i); + } + + nbt = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, nbt, Math.max(1493, i)); + if (i < SharedConstants.getCurrentVersion().getWorldVersion()) { + nbt.putInt("DataVersion", SharedConstants.getCurrentVersion().getWorldVersion()); + } + + return nbt; + } + static class SaveEntry { private CompoundTag data; private final CompletableFuture result = new CompletableFuture<>(); From e7d3b31cb653bc51a88ac5a6a7df44eea5f4a4b2 Mon Sep 17 00:00:00 2001 From: Tom Martin Date: Mon, 28 Mar 2022 01:38:13 +0100 Subject: [PATCH 2/3] Fix crash when vanilla dfu fails due to vanilla bug. See https://bugs.mojang.com/browse/MC-241172 as one example --- .../world/storage/RegionCubeIO.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java b/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java index e66a9a283..e321e38ee 100644 --- a/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java +++ b/src/main/java/io/github/opencubicchunks/cubicchunks/world/storage/RegionCubeIO.java @@ -380,14 +380,19 @@ private CompoundTag mergeOldNbt(CubePos pos, CompoundTag[] cubeTags) { int version = ChunkStorage.getVersion(cubeTags[i]); - cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], version, 1493); - //if (nbt.getCompound("Level").getBoolean("hasLegacyStructureData")) { - // if (this.legacyStructureHandler == null) { - // this.legacyStructureHandler = LegacyStructureDataHandler.getLegacyStructureHandler(worldKey, (DimensionDataStorage)persistentStateManagerFactory.get()); - // } - // nbt = this.legacyStructureHandler.updateFromLegacy(nbt); - //} - cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], Math.max(1493, version)); + try { + cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], version, 1493); + //if (nbt.getCompound("Level").getBoolean("hasLegacyStructureData")) { + // if (this.legacyStructureHandler == null) { + // this.legacyStructureHandler = LegacyStructureDataHandler.getLegacyStructureHandler(worldKey, (DimensionDataStorage)persistentStateManagerFactory.get()); + // } + // nbt = this.legacyStructureHandler.updateFromLegacy(nbt); + //} + cubeTags[i] = NbtUtils.update(this.dataFixer, DataFixTypes.CHUNK, cubeTags[i], Math.max(1493, version)); + } catch (Exception e) { + System.err.printf("Vanilla DFU error on (%d, %d, %d)!\n", pos.getX(), pos.getY(), pos.getZ()); + e.printStackTrace(System.err); + } if (cubeTags[i] == null) { LOGGER.warn("Dropping incomplete cube at " + pos); return null; From 77623594bf8797d8d3c4d5d8ca7e9ffbe37531d3 Mon Sep 17 00:00:00 2001 From: Tom Martin Date: Wed, 30 Mar 2022 18:31:56 +0100 Subject: [PATCH 3/3] Re-add databreaker commented out --- build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4bbade273..3d1656a71 100644 --- a/build.gradle +++ b/build.gradle @@ -237,7 +237,11 @@ dependencies { } // modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" -// + + // This is commented out as it (obviously) breaks loading old worlds, and should probably be enabled in dev env +// modRuntime ("supercoder79:databreaker:0.2.8") { +// exclude module: "fabric-loader" +// } compileOnly('com.google.code.findbugs:jsr305:3.0.1') testCompileOnly('com.google.code.findbugs:jsr305:3.0.1')