Skip to content

Commit

Permalink
feat: optional data converter shim for upgrading world data
Browse files Browse the repository at this point in the history
  • Loading branch information
@mworzala
mworzala committed Apr 26, 2024
1 parent 10d84dd commit 8cddebb
Show file tree
Hide file tree
Showing 6 changed files with 112 additions and 17 deletions.
2 changes: 2 additions & 0 deletions src/main/java/net/hollowcube/polar/AnvilPolar.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package net.hollowcube.polar;

import net.minestom.server.MinecraftServer;
import net.minestom.server.instance.Chunk;
import net.minestom.server.instance.Instance;
import net.minestom.server.instance.InstanceContainer;
Expand Down Expand Up @@ -95,6 +96,7 @@ public class AnvilPolar {

var world = new PolarWorld(
PolarWorld.LATEST_VERSION,
MinecraftServer.DATA_VERSION,
PolarWorld.DEFAULT_COMPRESSION,
(byte) minSection, (byte) maxSection,
new byte[0],
Expand Down
51 changes: 51 additions & 0 deletions src/main/java/net/hollowcube/polar/PolarDataConverter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package net.hollowcube.polar;

import net.kyori.adventure.nbt.CompoundBinaryTag;
import net.minestom.server.MinecraftServer;
import org.jetbrains.annotations.NotNull;

import java.util.Map;

/**
* Allows for upgrading world data from one game version to another.
*/
public interface PolarDataConverter {
@NotNull
PolarDataConverter NOOP = new PolarDataConverter() {
};

/**
* Returns the data version to use on worlds lower than {@link PolarWorld#VERSION_DATA_CONVERTER} which
* do not store a data version. Defaults to the current Minestom data version.
*/
default int defaultDataVersion() {
return MinecraftServer.DATA_VERSION;
}

/**
* Returns the current data version of the world.
*/
default int dataVersion() {
return MinecraftServer.DATA_VERSION;
}

/**
* <p>Converts the block palette from one version to another. Implementations are expected to modify
* the palette array in place.</p>
*
* @param palette An array of block namespaces, eg "minecraft:stone_stairs[facing=north]"
* @param fromVersion The data version of the palette
* @param toVersion The data version to convert the palette to
*/
default void convertBlockPalette(@NotNull String[] palette, int fromVersion, int toVersion) {

}

default @NotNull Map.Entry<String, CompoundBinaryTag> convertBlockEntityData(
@NotNull String id, @NotNull CompoundBinaryTag data,
int fromVersion, int toVersion
) {
return Map.entry(id, data);
}

}
16 changes: 11 additions & 5 deletions src/main/java/net/hollowcube/polar/PolarLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
import net.minestom.server.command.builder.arguments.minecraft.ArgumentBlockState;
import net.minestom.server.command.builder.exception.ArgumentSyntaxException;
import net.minestom.server.exception.ExceptionManager;
import net.minestom.server.instance.*;
import net.minestom.server.instance.Chunk;
import net.minestom.server.instance.IChunkLoader;
import net.minestom.server.instance.Instance;
import net.minestom.server.instance.Section;
import net.minestom.server.instance.block.Block;
import net.minestom.server.instance.block.BlockManager;
import net.minestom.server.network.NetworkBuffer;
import net.minestom.server.world.biomes.Biome;
import net.minestom.server.world.biomes.BiomeManager;
import net.minestom.server.world.biomes.VanillaBiome;
import org.jetbrains.annotations.Contract;
Expand All @@ -25,7 +27,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
Expand Down Expand Up @@ -236,8 +241,9 @@ private void loadBlockEntity(@NotNull PolarChunk.BlockEntity blockEntity, @NotNu
block = block.withHandler(BLOCK_MANAGER.getHandlerOrDummy(blockEntity.id()));
if (blockEntity.data() != null)
block = block.withNbt(blockEntity.data());
if (worldAccess != null)

chunk.setBlock(blockEntity.x(), blockEntity.y(), blockEntity.z(), block);
chunk.setBlock(blockEntity.x(), blockEntity.y(), blockEntity.z(), block);
}

// Unloading/saving
Expand Down Expand Up @@ -286,7 +292,7 @@ private void updateChunkData(@NotNull Short2ObjectMap<String> blockCache, @NotNu

var blockEntities = new ArrayList<PolarChunk.BlockEntity>();
var sections = new PolarSection[dimension.getHeight() / Chunk.CHUNK_SECTION_SIZE];
assert sections.length == chunk.getSections().size(): "World height mismatch";
assert sections.length == chunk.getSections().size() : "World height mismatch";

var heightmaps = new int[PolarChunk.MAX_HEIGHTMAPS][];

Expand Down
35 changes: 27 additions & 8 deletions src/main/java/net/hollowcube/polar/PolarReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ private PolarReader() {
}

public static @NotNull PolarWorld read(byte @NotNull [] data) {
return read(data, PolarDataConverter.NOOP);
}

public static @NotNull PolarWorld read(byte @NotNull [] data, @NotNull PolarDataConverter dataConverter) {
var buffer = new NetworkBuffer(ByteBuffer.wrap(data));
buffer.writeIndex(data.length); // Set write index to end so readableBytes returns remaining bytes

Expand All @@ -34,6 +38,10 @@ private PolarReader() {
short version = buffer.read(SHORT);
validateVersion(version);

int dataVersion = version >= PolarWorld.VERSION_DATA_CONVERTER
? buffer.read(VAR_INT)
: dataConverter.defaultDataVersion();

var compression = PolarWorld.CompressionType.fromId(buffer.read(BYTE));
assertThat(compression != null, "Invalid compression type");
var compressedDataLength = buffer.read(VAR_INT);
Expand All @@ -49,21 +57,21 @@ private PolarReader() {
if (version > PolarWorld.VERSION_WORLD_USERDATA)
userData = buffer.read(BYTE_ARRAY);

var chunks = buffer.readCollection(b -> readChunk(version, b, maxSection - minSection + 1), MAX_CHUNKS);
var chunks = buffer.readCollection(b -> readChunk(dataConverter, version, dataVersion, b, maxSection - minSection + 1), MAX_CHUNKS);

return new PolarWorld(version, compression, minSection, maxSection, userData, chunks);
return new PolarWorld(version, dataVersion, compression, minSection, maxSection, userData, chunks);
}

private static @NotNull PolarChunk readChunk(short version, @NotNull NetworkBuffer buffer, int sectionCount) {
private static @NotNull PolarChunk readChunk(@NotNull PolarDataConverter dataConverter, short version, int dataVersion, @NotNull NetworkBuffer buffer, int sectionCount) {
var chunkX = buffer.read(VAR_INT);
var chunkZ = buffer.read(VAR_INT);

var sections = new PolarSection[sectionCount];
for (int i = 0; i < sectionCount; i++) {
sections[i] = readSection(version, buffer);
sections[i] = readSection(dataConverter, version, dataVersion, buffer);
}

var blockEntities = buffer.readCollection(b -> readBlockEntity(version, b), MAX_BLOCK_ENTITIES);
var blockEntities = buffer.readCollection(b -> readBlockEntity(dataConverter, version, dataVersion, b), MAX_BLOCK_ENTITIES);

var heightmaps = new int[PolarChunk.MAX_HEIGHTMAPS][];
int heightmapMask = buffer.read(INT);
Expand Down Expand Up @@ -95,11 +103,14 @@ private PolarReader() {
);
}

private static @NotNull PolarSection readSection(short version, @NotNull NetworkBuffer buffer) {
private static @NotNull PolarSection readSection(@NotNull PolarDataConverter dataConverter, short version, int dataVersion, @NotNull NetworkBuffer buffer) {
// If section is empty exit immediately
if (buffer.read(BOOLEAN)) return new PolarSection();

var blockPalette = buffer.readCollection(STRING, MAX_BLOCK_PALETTE_SIZE).toArray(String[]::new);
if (dataVersion < dataConverter.dataVersion()) {
dataConverter.convertBlockPalette(blockPalette, dataVersion, dataConverter.dataVersion());
}
if (version <= PolarWorld.VERSION_SHORT_GRASS) {
for (int i = 0; i < blockPalette.length; i++) {
String strippedID = blockPalette[i].split("\\[")[0];
Expand Down Expand Up @@ -141,11 +152,11 @@ private PolarReader() {
return new PolarSection(blockPalette, blockData, biomePalette, biomeData, blockLight, skyLight);
}

private static @NotNull PolarChunk.BlockEntity readBlockEntity(int version, @NotNull NetworkBuffer buffer) {
private static @NotNull PolarChunk.BlockEntity readBlockEntity(@NotNull PolarDataConverter dataConverter, int version, int dataVersion, @NotNull NetworkBuffer buffer) {
int posIndex = buffer.read(INT);
var id = buffer.readOptional(STRING);

CompoundBinaryTag nbt = null;
CompoundBinaryTag nbt = CompoundBinaryTag.empty();
if (version <= PolarWorld.VERSION_USERDATA_OPT_BLOCK_ENT_NBT || buffer.read(BOOLEAN)) {
if (version <= PolarWorld.VERSION_MINESTOM_NBT_READ_BREAK || FORCE_LEGACY_NBT) {
nbt = (CompoundBinaryTag) legacyReadNBT(buffer);
Expand All @@ -154,6 +165,14 @@ private PolarReader() {
}
}

if (dataVersion < dataConverter.dataVersion()) {
var converted = dataConverter.convertBlockEntityData(id == null ? "" : id, nbt, dataVersion, dataConverter.dataVersion());
id = converted.getKey();
if (id.isEmpty()) id = null;
nbt = converted.getValue();
if (nbt.size() == 0) nbt = null;
}

return new PolarChunk.BlockEntity(
ChunkUtils.blockIndexToChunkPositionX(posIndex),
ChunkUtils.blockIndexToChunkPositionY(posIndex),
Expand Down
15 changes: 13 additions & 2 deletions src/main/java/net/hollowcube/polar/PolarWorld.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import net.minestom.server.MinecraftServer;
import net.minestom.server.utils.chunk.ChunkUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
Expand All @@ -15,18 +16,20 @@
@SuppressWarnings("UnstableApiUsage")
public class PolarWorld {
public static final int MAGIC_NUMBER = 0x506F6C72; // `Polr`
public static final short LATEST_VERSION = 5;
public static final short LATEST_VERSION = 6;

static final short VERSION_UNIFIED_LIGHT = 1;
static final short VERSION_USERDATA_OPT_BLOCK_ENT_NBT = 2;
static final short VERSION_MINESTOM_NBT_READ_BREAK = 3;
static final short VERSION_WORLD_USERDATA = 4;
static final short VERSION_SHORT_GRASS = 5; // >:(
static final short VERSION_DATA_CONVERTER = 6;

public static CompressionType DEFAULT_COMPRESSION = CompressionType.ZSTD;

// Polar metadata
private final short version;
private final int dataVersion;
private CompressionType compression;

// World metadata
Expand All @@ -38,17 +41,19 @@ public class PolarWorld {
private final Long2ObjectMap<PolarChunk> chunks = new Long2ObjectOpenHashMap<>();

public PolarWorld() {
this(LATEST_VERSION, DEFAULT_COMPRESSION, (byte) -4, (byte) 19, new byte[0], List.of());
this(LATEST_VERSION, MinecraftServer.DATA_VERSION, DEFAULT_COMPRESSION, (byte) -4, (byte) 19, new byte[0], List.of());
}

public PolarWorld(
short version,
int dataVersion,
@NotNull CompressionType compression,
byte minSection, byte maxSection,
byte @NotNull [] userData,
@NotNull List<PolarChunk> chunks
) {
this.version = version;
this.dataVersion = dataVersion;
this.compression = compression;

this.minSection = minSection;
Expand All @@ -65,9 +70,14 @@ public short version() {
return version;
}

public int dataVersion() {
return dataVersion;
}

public @NotNull CompressionType compression() {
return compression;
}

public void setCompression(@NotNull CompressionType compression) {
this.compression = compression;
}
Expand All @@ -91,6 +101,7 @@ public void userData(byte @NotNull [] userData) {
public @Nullable PolarChunk chunkAt(int x, int z) {
return chunks.getOrDefault(ChunkUtils.getChunkIndex(x, z), null);
}

public void updateChunkAt(int x, int z, @NotNull PolarChunk chunk) {
chunks.put(ChunkUtils.getChunkIndex(x, z), chunk);
}
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/net/hollowcube/polar/PolarWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,26 @@

@SuppressWarnings("UnstableApiUsage")
public class PolarWriter {
private PolarWriter() {}
private PolarWriter() {
}

public static byte[] write(@NotNull PolarWorld world) {
return write(world, PolarDataConverter.NOOP);
}

public static byte[] write(@NotNull PolarWorld world, @NotNull PolarDataConverter dataConverter) {
// Write the compressed content first
var content = new NetworkBuffer(ByteBuffer.allocate(1024));
content.write(BYTE, world.minSection());
content.write(BYTE, world.maxSection());
content.write(BYTE_ARRAY, world.userData());
content.writeCollection(world.chunks(), (b , c) -> writeChunk(b, c, world.maxSection() - world.minSection() + 1));
content.writeCollection(world.chunks(), (b, c) -> writeChunk(b, c, world.maxSection() - world.minSection() + 1));

// Create final buffer
return NetworkBuffer.makeArray(buffer -> {
buffer.write(INT, PolarWorld.MAGIC_NUMBER);
buffer.write(SHORT, PolarWorld.LATEST_VERSION);
buffer.write(VAR_INT, dataConverter.dataVersion());
buffer.write(BYTE, (byte) world.compression().ordinal());
switch (world.compression()) {
case NONE -> {
Expand Down

0 comments on commit 8cddebb

Please sign in to comment.