From 1624b09640e2ff5cbbc8475cba98605d837cf5bd Mon Sep 17 00:00:00 2001 From: Patrick Owen Date: Wed, 10 Jan 2024 23:29:49 -0500 Subject: [PATCH 1/2] Switch SerializableVoxelData from a Vec to a Vec --- client/src/sim.rs | 3 +-- common/src/node.rs | 37 ++++++++++++++------------ common/src/proto.rs | 7 ++--- common/src/world.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++ server/src/sim.rs | 2 +- 5 files changed, 91 insertions(+), 23 deletions(-) diff --git a/client/src/sim.rs b/client/src/sim.rs index 7bdeead0..14c27113 100644 --- a/client/src/sim.rs +++ b/client/src/sim.rs @@ -331,8 +331,7 @@ impl Sim { } } for (chunk_id, voxel_data) in msg.modified_chunks { - let Some(voxel_data) = VoxelData::from_serializable(&voxel_data, self.cfg.chunk_size) - else { + let Some(voxel_data) = VoxelData::deserialize(&voxel_data, self.cfg.chunk_size) else { tracing::error!("Voxel data received from server is of incorrect dimension"); continue; }; diff --git a/common/src/node.rs b/common/src/node.rs index 5f39f6d4..a840bc5d 100644 --- a/common/src/node.rs +++ b/common/src/node.rs @@ -8,7 +8,7 @@ use crate::collision_math::Ray; use crate::dodeca::Vertex; use crate::graph::{Graph, NodeId}; use crate::lru_slab::SlotId; -use crate::proto::{BlockUpdate, Position, SerializableVoxelData}; +use crate::proto::{BlockUpdate, Position, SerializedVoxelData}; use crate::world::Material; use crate::worldgen::NodeState; use crate::{math, Chunks}; @@ -326,46 +326,49 @@ impl VoxelData { } } - /// Returns a `VoxelData` with void margins based on the given `SerializableVoxelData`, or `None` if - /// the `SerializableVoxelData` came from a `VoxelData` with the wrong dimension. - pub fn from_serializable(serializable: &SerializableVoxelData, dimension: u8) -> Option { - if serializable.voxels.len() != usize::from(dimension).pow(3) { + /// Returns a `VoxelData` with void margins based on the given `SerializedVoxelData`, or `None` if + /// the `SerializedVoxelData` came from a `VoxelData` with the wrong dimension or an unknown material. + pub fn deserialize(serialized: &SerializedVoxelData, dimension: u8) -> Option { + if serialized.inner.len() != usize::from(dimension).pow(3) * 2 { return None; } + let mut materials = serialized + .inner + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])); + let mut data = vec![Material::Void; (usize::from(dimension) + 2).pow(3)]; - let mut input_index = 0; for z in 0..dimension { for y in 0..dimension { for x in 0..dimension { - // We cannot use a linear copy here because `data` has margins, while `serializable.voxels` does not. - data[Coords([x, y, z]).to_index(dimension)] = serializable.voxels[input_index]; - input_index += 1; + // We cannot use a linear copy here because `data` has margins, while `serialized.inner` does not. + data[Coords([x, y, z]).to_index(dimension)] = + materials.next().unwrap().try_into().ok()?; } } } Some(VoxelData::Dense(data.into_boxed_slice())) } - /// Returns a `SerializableVoxelData` corresponding to `self`. Assumes that`self` is `Dense` and + /// Returns a `SerializedVoxelData` corresponding to `self`. Assumes that `self` is `Dense` and /// has the right dimension, as it will panic or return incorrect data otherwise. - pub fn to_serializable(&self, dimension: u8) -> SerializableVoxelData { + pub fn serialize(&self, dimension: u8) -> SerializedVoxelData { let VoxelData::Dense(data) = self else { panic!("Only dense chunks can be serialized."); }; - let mut serializable: Vec = Vec::with_capacity(usize::from(dimension).pow(3)); + let mut serialized: Vec = Vec::with_capacity(usize::from(dimension).pow(3) * 2); for z in 0..dimension { for y in 0..dimension { for x in 0..dimension { - // We cannot use a linear copy here because `data` has margins, while `serializable.voxels` does not. - serializable.push(data[Coords([x, y, z]).to_index(dimension)]); + // We cannot use a linear copy here because `data` has margins, while `serialized.inner` does not. + serialized + .extend((data[Coords([x, y, z]).to_index(dimension)] as u16).to_le_bytes()); } } } - SerializableVoxelData { - voxels: serializable, - } + SerializedVoxelData { inner: serialized } } } diff --git a/common/src/proto.rs b/common/src/proto.rs index 292383ce..31641828 100644 --- a/common/src/proto.rs +++ b/common/src/proto.rs @@ -57,7 +57,7 @@ pub struct Spawns { pub despawns: Vec, pub nodes: Vec, pub block_updates: Vec, - pub modified_chunks: Vec<(ChunkId, SerializableVoxelData)>, + pub modified_chunks: Vec<(ChunkId, SerializedVoxelData)>, } #[derive(Debug, Serialize, Deserialize)] @@ -84,8 +84,9 @@ pub struct BlockUpdate { } #[derive(Debug, Serialize, Deserialize)] -pub struct SerializableVoxelData { - pub voxels: Vec, +pub struct SerializedVoxelData { + /// Dense 3D array of 16-bit material tags for all voxels in this chunk + pub inner: Vec, } #[derive(Debug, Serialize, Deserialize)] diff --git a/common/src/world.rs b/common/src/world.rs index 7c4acc75..b1580c26 100644 --- a/common/src/world.rs +++ b/common/src/world.rs @@ -51,3 +51,68 @@ pub enum Material { impl Material { pub const COUNT: usize = 40; } + +impl TryFrom for Material { + type Error = (); + + fn try_from(value: u16) -> Result { + Ok(match value { + 0 => Material::Void, + 1 => Material::Dirt, + 2 => Material::Sand, + 3 => Material::Silt, + 4 => Material::Clay, + 5 => Material::Mud, + 6 => Material::SandyLoam, + 7 => Material::SiltyLoam, + 8 => Material::ClayLoam, + 9 => Material::RedSand, + 10 => Material::Limestone, + 11 => Material::Shale, + 12 => Material::Dolomite, + 13 => Material::Sandstone, + 14 => Material::RedSandstone, + 15 => Material::Marble, + 16 => Material::Slate, + 17 => Material::Granite, + 18 => Material::Diorite, + 19 => Material::Andesite, + 20 => Material::Gabbro, + 21 => Material::Basalt, + 22 => Material::Olivine, + 23 => Material::Water, + 24 => Material::Lava, + 25 => Material::Wood, + 26 => Material::Leaves, + 27 => Material::WoodPlanks, + 28 => Material::GreyBrick, + 29 => Material::WhiteBrick, + 30 => Material::Ice, + 31 => Material::IceSlush, + 32 => Material::Gravel, + 33 => Material::Snow, + 34 => Material::CoarseGrass, + 35 => Material::TanGrass, + 36 => Material::LushGrass, + 37 => Material::MudGrass, + 38 => Material::Grass, + 39 => Material::CaveGrass, + _ => Err(())?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::Material; + + #[test] + fn u16_to_material_consistency_check() { + for i in 0..Material::COUNT { + let index = u16::try_from(i).unwrap(); + let material = + Material::try_from(index).expect("no missing entries in try_from match statement"); + assert_eq!(index, material as u16); + } + } +} diff --git a/server/src/sim.rs b/server/src/sim.rs index 8ab95608..0224d5fc 100644 --- a/server/src/sim.rs +++ b/server/src/sim.rs @@ -208,7 +208,7 @@ impl Sim { spawns .modified_chunks - .push((chunk_id, voxels.to_serializable(self.cfg.chunk_size))); + .push((chunk_id, voxels.serialize(self.cfg.chunk_size))); } spawns } From f8a02ac27c731a38bb0829fccac63a6e9270d731 Mon Sep 17 00:00:00 2001 From: Patrick Owen Date: Wed, 10 Jan 2024 23:35:13 -0500 Subject: [PATCH 2/2] Save and load all modified chunks --- client/src/graphics/voxels/mod.rs | 2 +- client/src/sim.rs | 9 +-- common/src/graph.rs | 5 ++ common/src/proto.rs | 2 +- save/src/lib.rs | 10 ++++ server/src/lib.rs | 4 +- server/src/sim.rs | 96 +++++++++++++++++++++++++++---- 7 files changed, 110 insertions(+), 18 deletions(-) diff --git a/client/src/graphics/voxels/mod.rs b/client/src/graphics/voxels/mod.rs index 9f9d1d34..571f4010 100644 --- a/client/src/graphics/voxels/mod.rs +++ b/client/src/graphics/voxels/mod.rs @@ -105,7 +105,7 @@ impl Voxels { // Now that the block is populated, we can apply any pending block updates the server // provided that the client couldn't apply. - if let Some(block_updates) = sim.pending_modified_chunks.remove(&chunk_id) { + if let Some(block_updates) = sim.preloaded_block_updates.remove(&chunk_id) { for block_update in block_updates { // The chunk was just populated, so a block update should always succeed. assert!(sim.graph.update_block(&block_update)); diff --git a/client/src/sim.rs b/client/src/sim.rs index 14c27113..9498734e 100644 --- a/client/src/sim.rs +++ b/client/src/sim.rs @@ -38,7 +38,8 @@ const MATERIAL_PALETTE: [Material; 10] = [ pub struct Sim { // World state pub graph: Graph, - pub pending_modified_chunks: FxHashMap>, + /// Voxel data that have been downloaded from the server for chunks not yet introduced to the graph + pub preloaded_block_updates: FxHashMap>, pub graph_entities: GraphEntities, entity_ids: FxHashMap, pub world: hecs::World, @@ -84,7 +85,7 @@ impl Sim { populate_fresh_nodes(&mut graph); Self { graph, - pending_modified_chunks: FxHashMap::default(), + preloaded_block_updates: FxHashMap::default(), graph_entities: GraphEntities::new(), entity_ids: FxHashMap::default(), world: hecs::World::new(), @@ -324,13 +325,13 @@ impl Sim { populate_fresh_nodes(&mut self.graph); for block_update in msg.block_updates.into_iter() { if !self.graph.update_block(&block_update) { - self.pending_modified_chunks + self.preloaded_block_updates .entry(block_update.chunk_id) .or_default() .push(block_update); } } - for (chunk_id, voxel_data) in msg.modified_chunks { + for (chunk_id, voxel_data) in msg.voxel_data { let Some(voxel_data) = VoxelData::deserialize(&voxel_data, self.cfg.chunk_size) else { tracing::error!("Voxel data received from server is of incorrect dimension"); continue; diff --git a/common/src/graph.rs b/common/src/graph.rs index 5a0c764d..ab2922ae 100644 --- a/common/src/graph.rs +++ b/common/src/graph.rs @@ -226,6 +226,11 @@ impl Graph { node.0 } + #[inline] + pub fn from_hash(&self, hash: u128) -> NodeId { + NodeId(hash) + } + /// Ensure all shorter neighbors of a not-yet-created child node exist and return them, excluding the given parent node fn populate_shorter_neighbors_of_child( &mut self, diff --git a/common/src/proto.rs b/common/src/proto.rs index 31641828..2c63758b 100644 --- a/common/src/proto.rs +++ b/common/src/proto.rs @@ -57,7 +57,7 @@ pub struct Spawns { pub despawns: Vec, pub nodes: Vec, pub block_updates: Vec, - pub modified_chunks: Vec<(ChunkId, SerializedVoxelData)>, + pub voxel_data: Vec<(ChunkId, SerializedVoxelData)>, } #[derive(Debug, Serialize, Deserialize)] diff --git a/save/src/lib.rs b/save/src/lib.rs index 3e69625c..3de042c6 100644 --- a/save/src/lib.rs +++ b/save/src/lib.rs @@ -139,6 +139,16 @@ impl Reader<'_> { .map_err(GetError::DecompressionFailed)?; Ok(Some(Character::decode(&*self.accum)?)) } + + /// Temporary function to load all voxel-related save data at once. + /// TODO: Replace this implementation with a streaming implementation + /// that does not require loading everything at once + pub fn get_all_voxel_node_ids(&self) -> Result, GetError> { + self.voxel_nodes + .iter()? + .map(|n| Ok(n.map_err(GetError::from)?.0.value())) + .collect() + } } fn decompress( diff --git a/server/src/lib.rs b/server/src/lib.rs index eff613f2..dcf6e49d 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -56,7 +56,7 @@ impl Server { fn new(params: SimConfig, save: Save) -> Self { let cfg = Arc::new(params); Self { - sim: Sim::new(cfg.clone()), + sim: Sim::new(cfg.clone(), &save), cfg, clients: DenseSlotMap::default(), save, @@ -125,7 +125,7 @@ impl Server { || !spawns.despawns.is_empty() || !spawns.nodes.is_empty() || !spawns.block_updates.is_empty() - || !spawns.modified_chunks.is_empty() + || !spawns.voxel_data.is_empty() { handles.ordered.try_send(spawns.clone()) } else { diff --git a/server/src/sim.rs b/server/src/sim.rs index 0224d5fc..d6b8c948 100644 --- a/server/src/sim.rs +++ b/server/src/sim.rs @@ -1,6 +1,9 @@ use std::sync::Arc; -use common::proto::BlockUpdate; +use anyhow::Context; +use common::dodeca::Vertex; +use common::node::VoxelData; +use common::proto::{BlockUpdate, SerializedVoxelData}; use common::{node::ChunkId, GraphEntities}; use fxhash::{FxHashMap, FxHashSet}; use hecs::Entity; @@ -32,29 +35,40 @@ pub struct Sim { entity_ids: FxHashMap, world: hecs::World, graph: Graph, + /// Voxel data that has been fetched from a savefile but not yet introduced to the graph + preloaded_voxel_data: FxHashMap, spawns: Vec, despawns: Vec, graph_entities: GraphEntities, + /// All nodes that have entity-related information yet to be saved dirty_nodes: FxHashSet, + /// All nodes that have voxel-related information yet to be saved + dirty_voxel_nodes: FxHashSet, + /// All chunks that have ever had any block updates applied to them and can no longer be regenerated with worldgen modified_chunks: FxHashSet, } impl Sim { - pub fn new(cfg: Arc) -> Self { + pub fn new(cfg: Arc, save: &save::Save) -> Self { let mut result = Self { rng: SmallRng::from_entropy(), step: 0, entity_ids: FxHashMap::default(), world: hecs::World::new(), graph: Graph::new(cfg.chunk_size), + preloaded_voxel_data: FxHashMap::default(), spawns: Vec::new(), despawns: Vec::new(), graph_entities: GraphEntities::new(), dirty_nodes: FxHashSet::default(), + dirty_voxel_nodes: FxHashSet::default(), modified_chunks: FxHashSet::default(), cfg, }; + result + .load_all_voxels(save) + .expect("save file must be of a valid format"); ensure_nearby( &mut result.graph, &Position::origin(), @@ -86,16 +100,45 @@ impl Sim { } let dirty_nodes = self.dirty_nodes.drain().collect::>(); + let dirty_voxel_nodes = self.dirty_voxel_nodes.drain().collect::>(); for node in dirty_nodes { let entities = self.snapshot_node(node); writer.put_entity_node(self.graph.hash_of(node), &entities)?; } + for node in dirty_voxel_nodes { + let voxels = self.snapshot_voxel_node(node); + writer.put_voxel_node(self.graph.hash_of(node), &voxels)?; + } drop(writer); tx.commit()?; Ok(()) } + fn load_all_voxels(&mut self, save: &save::Save) -> anyhow::Result<()> { + let read_guard = save.read()?; + let mut read = read_guard.get()?; + for node_hash in read.get_all_voxel_node_ids()? { + let Some(voxel_node) = read.get_voxel_node(node_hash)? else { + continue; + }; + for chunk in voxel_node.chunks { + let voxels = SerializedVoxelData { + inner: chunk.voxels, + }; + let vertex = Vertex::iter() + .nth(chunk.vertex as usize) + .context("deserializing vertex ID")?; + self.preloaded_voxel_data.insert( + ChunkId::new(self.graph.from_hash(node_hash), vertex), + VoxelData::deserialize(&voxels, self.cfg.chunk_size) + .context("deserializing voxel data")?, + ); + } + } + Ok(()) + } + fn snapshot_node(&self, node: NodeId) -> save::EntityNode { let mut entities = Vec::new(); for &entity in self.graph_entities.get(node) { @@ -131,6 +174,24 @@ impl Sim { save::EntityNode { entities } } + fn snapshot_voxel_node(&self, node: NodeId) -> save::VoxelNode { + let mut chunks = vec![]; + let node_data = self.graph.get(node).as_ref().unwrap(); + for vertex in Vertex::iter() { + if !self.modified_chunks.contains(&ChunkId::new(node, vertex)) { + continue; + } + let Chunk::Populated { ref voxels, .. } = node_data.chunks[vertex] else { + panic!("Unknown chunk listed as modified"); + }; + chunks.push(save::Chunk { + vertex: vertex as u32, + voxels: voxels.serialize(self.cfg.chunk_size).inner, + }) + } + save::VoxelNode { chunks } + } + pub fn spawn_character(&mut self, hello: ClientHello) -> (EntityId, Entity) { let id = self.new_id(); info!(%id, name = %hello.name, "spawning character"); @@ -194,7 +255,7 @@ impl Sim { .map(|(side, parent)| FreshNode { side, parent }) .collect(), block_updates: Vec::new(), - modified_chunks: Vec::new(), + voxel_data: Vec::new(), }; for (entity, &id) in &mut self.world.query::<&EntityId>() { spawns.spawns.push((id, dump_entity(&self.world, entity))); @@ -207,7 +268,7 @@ impl Sim { }; spawns - .modified_chunks + .voxel_data .push((chunk_id, voxels.serialize(self.cfg.chunk_size))); } spawns @@ -245,6 +306,9 @@ impl Sim { ensure_nearby(&mut self.graph, position, f64::from(self.cfg.view_distance)); } + let fresh_nodes = self.graph.fresh().to_vec(); + populate_fresh_nodes(&mut self.graph); + let mut accepted_block_updates: Vec = vec![]; for block_update in pending_block_updates.into_iter() { @@ -252,6 +316,7 @@ impl Sim { tracing::warn!("Block update received from ungenerated chunk"); } self.modified_chunks.insert(block_update.chunk_id); + self.dirty_voxel_nodes.insert(block_update.chunk_id.node); accepted_block_updates.push(block_update); } @@ -261,16 +326,28 @@ impl Sim { let id = *self.world.get::<&EntityId>(entity).unwrap(); spawns.push((id, dump_entity(&self.world, entity))); } - if !self.graph.fresh().is_empty() { + + let mut fresh_voxel_data = vec![]; + for fresh_node in fresh_nodes.iter().copied() { + for vertex in Vertex::iter() { + let chunk = ChunkId::new(fresh_node, vertex); + if let Some(voxel_data) = self.preloaded_voxel_data.remove(&chunk) { + fresh_voxel_data.push((chunk, voxel_data.serialize(self.cfg.chunk_size))); + self.modified_chunks.insert(chunk); + self.graph.populate_chunk(chunk, voxel_data, true) + } + } + } + + if !fresh_nodes.is_empty() { trace!(count = self.graph.fresh().len(), "broadcasting fresh nodes"); } + let spawns = Spawns { step: self.step, spawns, despawns: std::mem::take(&mut self.despawns), - nodes: self - .graph - .fresh() + nodes: fresh_nodes .iter() .filter_map(|&id| { let side = self.graph.parent(id)?; @@ -281,9 +358,8 @@ impl Sim { }) .collect(), block_updates: accepted_block_updates, - modified_chunks: vec![], + voxel_data: fresh_voxel_data, }; - populate_fresh_nodes(&mut self.graph); // We want to load all chunks that a player can interact with in a single step, so chunk_generation_distance // is set up to cover that distance.