From 9cf8951472ae973c5a2f0cdd0515fb35b57f1ac5 Mon Sep 17 00:00:00 2001 From: Nebual Date: Sun, 14 Apr 2013 04:27:11 -0700 Subject: [PATCH] Saving: Added Region Save Mode A very scalable system. Sectors (8x8x8 blocks) are stored 4x4x4 in headerless files, 2 bytes per block. Position data is not stored, but inferred based on the byte offset within the file. Whole regions (32x32x32 blocks) are read at once when world._show_sector asks. Sectors can be written to individually, though without a dirty_sectors system, it currently writes all sectors in memory to disk during a save. --- controllers.py | 1 + globals.py | 6 ++-- savingsystem.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++-- world.py | 20 ++++++++++++- 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/controllers.py b/controllers.py index 53cdb142..9f90e866 100644 --- a/controllers.py +++ b/controllers.py @@ -195,6 +195,7 @@ def setup(self): self.model = Model() self.player = Player((0, 0, 0), (-20, 0), game_mode=G.GAME_MODE) + self.save_to_file() #So the hardcoded spawn sectors aren't overwritten by the worldgen print('Game mode: ' + self.player.game_mode) self.item_list = ItemSelector(self, self.player, self.model) self.inventory_list = InventorySelector(self, self.player, self.model) diff --git a/globals.py b/globals.py index ab4112db..63347b93 100644 --- a/globals.py +++ b/globals.py @@ -91,10 +91,11 @@ PICKLE_SAVE_MODE = 'pickle' PICKLE_COMPRESSED_SAVE_MODE = 'pickle_compressed' FLATFILE_SAVE_MODE = 'flatfile' +REGION_SAVE_MODE = 'region' SAVE_MODES = ( - PICKLE_SAVE_MODE, PICKLE_COMPRESSED_SAVE_MODE, FLATFILE_SAVE_MODE + PICKLE_SAVE_MODE, PICKLE_COMPRESSED_SAVE_MODE, FLATFILE_SAVE_MODE, REGION_SAVE_MODE ) -SAVE_MODE = FLATFILE_SAVE_MODE +SAVE_MODE = REGION_SAVE_MODE # @@ -171,6 +172,7 @@ MAX_FPS = 60 # Maximum frames per second. VISIBLE_SECTORS_RADIUS = 8 +DELOAD_SECTORS_RADIUS = 16 DRAW_DISTANCE_CHOICES = { 'short': 60.0, diff --git a/savingsystem.py b/savingsystem.py index aa1fe4ed..37052964 100644 --- a/savingsystem.py +++ b/savingsystem.py @@ -23,6 +23,24 @@ structuchar2 = struct.Struct("BB") structvecBB = struct.Struct("hhhBB") +null2 = struct.pack("xx") #Two \0's +null1024 = null2*512 #1024 \0's +air = G.BLOCKS_DIR[(0,0)] + +def sector_to_filename(secpos): + x,y,z = secpos + return "%i.%i.%i.pyr" % (x/4, y/4, z/4) +def region_to_filename(region): + return "%i.%i.%i.pyr" % region +def sector_to_region(secpos): + x,y,z = secpos + return (x/4, y/4, z/4) +def sector_to_offset(secpos): + x,y,z = secpos + return ((x % 4)*16 + (y % 4)*4 + (z % 4)) * 1024 +def sector_to_blockpos(secpos): + x,y,z = secpos + return x*8, y*8, z*8 @performance_info def save_world(window, game_dir, world=None): @@ -35,7 +53,30 @@ def save_world(window, game_dir, world=None): pickle.dump(save, open(os.path.join(game_dir, world, "save.pkl"), "wb")) #blocks and sectors (window.model and window.model.sectors) - if G.SAVE_MODE == G.FLATFILE_SAVE_MODE: + if G.SAVE_MODE == G.REGION_SAVE_MODE: + #Saves individual sectors in region files (4x4x4 sectors) + blocks = window.model + for secpos in window.model.sectors: #TODO: only save dirty sectors + if not window.model.sectors[secpos]: + continue #Skip writing empty sectors + file = os.path.join(game_dir, world, sector_to_filename(secpos)) + if not os.path.exists(file): + with open(file, "w") as f: + f.truncate(64*1024) #Preallocate the file to be 64kb + with open(file, "rb+") as f: #Load up the region file + f.seek(sector_to_offset(secpos)) #Seek to the sector offset + cx, cy, cz = sector_to_blockpos(secpos) + fstr = "" + for x in xrange(cx, cx+8): + for y in xrange(cy, cy+8): + for z in xrange(cz, cz+8): + blk = blocks.get((x,y,z), air).id + if blk: + fstr += structuchar2.pack(blk.main, blk.sub) + else: + fstr += null2 + f.write(fstr) + elif G.SAVE_MODE == G.FLATFILE_SAVE_MODE: blocks = window.model with open(os.path.join(game_dir, world, "blocks.dat"), "wb", 1024*1024) as f: f.write(struct.pack("Q",len(blocks))) @@ -69,6 +110,38 @@ def remove_world(game_dir, world=None): import shutil shutil.rmtree(os.path.join(game_dir, world)) +def sector_exists(sector, world=None): + if world is None: world = "world" + return os.path.lexists(os.path.join(G.game_dir, world, sector_to_filename(sector))) + +def load_region(model, world=None, region=None, sector=None): + if world is None: world = "world" + sectors = model.sectors + blocks = model + SECTOR_SIZE = G.SECTOR_SIZE + BLOCKS_DIR = G.BLOCKS_DIR + if sector: region = sector_to_region(sector) + rx,ry,rz = region + rx,ry,rz = rx*32, ry*32, rz*32 + with open(os.path.join(G.game_dir, world, region_to_filename(region)), "rb") as f: + #Load every chunk in this region (4x4x4) + for cx in xrange(rx, rx+32, 8): + for cy in xrange(ry, ry+32, 8): + for cz in xrange(rz, rz+32, 8): + #Now load every block in this chunk (8x8x8) + fstr = f.read(1024) + if fstr != null1024: + fpos = 0 + for x in xrange(cx, cx+8): + for y in xrange(cy, cy+8): + for z in xrange(cz, cz+8): + read = fstr[fpos:fpos+2] + fpos += 2 + if read != null2: + position = x,y,z + blocks[position] = BLOCKS_DIR[structuchar2.unpack(read)] + sectors[(x/SECTOR_SIZE, y/SECTOR_SIZE, z/SECTOR_SIZE)].append(position) + @performance_info def open_world(gamecontroller, game_dir, world=None): if world is None: world = "world" @@ -79,8 +152,11 @@ def open_world(gamecontroller, game_dir, world=None): if loaded_save[0] == 3: #Version 3 if isinstance(loaded_save[1], Player): gamecontroller.player = loaded_save[1] if isinstance(loaded_save[2], float): gamecontroller.time_of_day = loaded_save[2] + #blocks and sectors (window.model and window.model.sectors) - if G.SAVE_MODE == G.FLATFILE_SAVE_MODE: + if G.SAVE_MODE == G.REGION_SAVE_MODE: + pass #Sectors are loaded by world._show_sector + elif G.SAVE_MODE == G.FLATFILE_SAVE_MODE: sectors = gamecontroller.model.sectors blocks = gamecontroller.model SECTOR_SIZE = G.SECTOR_SIZE diff --git a/world.py b/world.py index 9d53ae9e..434f6457 100644 --- a/world.py +++ b/world.py @@ -105,6 +105,8 @@ def __init__(self): self.transparency_batch = pyglet.graphics.Batch() self.group = TextureGroup(os.path.join('resources', 'textures', 'texture.png')) + import savingsystem #This module doesn't like being imported at modulescope + self.savingsystem = savingsystem self.shown = {} self._shown = {} self.sectors = defaultdict(list) @@ -270,7 +272,23 @@ def show_sector(self, sector, immediate=False): self.enqueue(self._show_sector, sector, urgent=True) def _show_sector(self, sector): - for position in self.sectors.get(sector, ()): + if G.SAVE_MODE == G.REGION_SAVE_MODE and not sector in self.sectors: + #The sector is not in memory, load or create it + if self.savingsystem.sector_exists(sector): + #If its on disk, load it + self.savingsystem.load_region(self, sector=sector) + else: + #The sector doesn't exist yet, generate it! + #self.generate_region(key) #<-- TODO + + #Temporary region generation function to show that the world grows + cx, cy, cz = self.savingsystem.sector_to_blockpos(sector) + rx, ry, rz = cx/32*32, cy/32*32, cz/32*32 # + for x in xrange(rx, rx+32): + for z in xrange(rz, rz+32): + self.init_block((x, ry, z), grass_block) #Flat layer of grass at the base of the region + + for position in self.sectors[sector]: if position not in self.shown and self.is_exposed(position): self.show_block(position)