Saving: Added Region Save Mode

A very scalable system. Sectors (8x8x8 blocks) are stored 4x4x4 in headerless
files, 2 bytes per block. Position data is not stored, but inferred based on
the byte offset within the file.

Whole regions (32x32x32 blocks) are read at once when world._show_sector asks.
Sectors can be written to individually, though without a dirty_sectors system,
it currently writes all sectors in memory to disk during a save.

controllers.py

@@ -195,6 +195,7 @@ def setup(self):
             self.model = Model()
             self.player = Player((0, 0, 0), (-20, 0),
                                  game_mode=G.GAME_MODE)
+            self.save_to_file() #So the hardcoded spawn sectors aren't overwritten by the worldgen
         print('Game mode: ' + self.player.game_mode)
         self.item_list = ItemSelector(self, self.player, self.model)
         self.inventory_list = InventorySelector(self, self.player, self.model)

globals.py

@@ -91,10 +91,11 @@
 PICKLE_SAVE_MODE = 'pickle'
 PICKLE_COMPRESSED_SAVE_MODE = 'pickle_compressed'
 FLATFILE_SAVE_MODE = 'flatfile'
+REGION_SAVE_MODE = 'region'
 SAVE_MODES = (
-    PICKLE_SAVE_MODE, PICKLE_COMPRESSED_SAVE_MODE, FLATFILE_SAVE_MODE
+    PICKLE_SAVE_MODE, PICKLE_COMPRESSED_SAVE_MODE, FLATFILE_SAVE_MODE, REGION_SAVE_MODE
 )
-SAVE_MODE = FLATFILE_SAVE_MODE
+SAVE_MODE = REGION_SAVE_MODE
 
 
 #
@@ -171,6 +172,7 @@
 MAX_FPS = 60  # Maximum frames per second.
 
 VISIBLE_SECTORS_RADIUS = 8
+DELOAD_SECTORS_RADIUS = 16
 
 DRAW_DISTANCE_CHOICES = {
     'short': 60.0,

savingsystem.py

@@ -23,6 +23,24 @@
 structuchar2 = struct.Struct("BB")
 structvecBB = struct.Struct("hhhBB")
 
+null2 = struct.pack("xx") #Two \0's
+null1024 = null2*512      #1024 \0's
+air = G.BLOCKS_DIR[(0,0)]
+
+def sector_to_filename(secpos):
+    x,y,z = secpos
+    return "%i.%i.%i.pyr" % (x/4, y/4, z/4)
+def region_to_filename(region):
+    return "%i.%i.%i.pyr" % region
+def sector_to_region(secpos):
+    x,y,z = secpos
+    return (x/4, y/4, z/4)
+def sector_to_offset(secpos):
+    x,y,z = secpos
+    return ((x % 4)*16 + (y % 4)*4 + (z % 4)) * 1024
+def sector_to_blockpos(secpos):
+    x,y,z = secpos
+    return x*8, y*8, z*8
 
 @performance_info
 def save_world(window, game_dir, world=None):
@@ -35,7 +53,30 @@ def save_world(window, game_dir, world=None):
     pickle.dump(save, open(os.path.join(game_dir, world, "save.pkl"), "wb"))
 
     #blocks and sectors (window.model and window.model.sectors)
-    if G.SAVE_MODE == G.FLATFILE_SAVE_MODE:
+    if G.SAVE_MODE == G.REGION_SAVE_MODE:
+        #Saves individual sectors in region files (4x4x4 sectors)
+        blocks = window.model
+        for secpos in window.model.sectors: #TODO: only save dirty sectors
+            if not window.model.sectors[secpos]:
+                continue #Skip writing empty sectors
+            file = os.path.join(game_dir, world, sector_to_filename(secpos)) 
+            if not os.path.exists(file):
+                with open(file, "w") as f:
+                    f.truncate(64*1024) #Preallocate the file to be 64kb
+            with open(file, "rb+") as f: #Load up the region file
+                f.seek(sector_to_offset(secpos)) #Seek to the sector offset
+                cx, cy, cz = sector_to_blockpos(secpos)
+                fstr = ""
+                for x in xrange(cx, cx+8):
+                    for y in xrange(cy, cy+8):
+                        for z in xrange(cz, cz+8):
+                            blk = blocks.get((x,y,z), air).id
+                            if blk:
+                                fstr += structuchar2.pack(blk.main, blk.sub)
+                            else:
+                                fstr += null2
+                f.write(fstr)
+    elif G.SAVE_MODE == G.FLATFILE_SAVE_MODE:
         blocks = window.model
         with open(os.path.join(game_dir, world, "blocks.dat"), "wb", 1024*1024) as f:
             f.write(struct.pack("Q",len(blocks)))
@@ -69,6 +110,38 @@ def remove_world(game_dir, world=None):
         import shutil
         shutil.rmtree(os.path.join(game_dir, world))
 
+def sector_exists(sector, world=None):
+    if world is None: world = "world"
+    return os.path.lexists(os.path.join(G.game_dir, world, sector_to_filename(sector)))
+
+def load_region(model, world=None, region=None, sector=None):
+    if world is None: world = "world"
+    sectors = model.sectors
+    blocks = model
+    SECTOR_SIZE = G.SECTOR_SIZE
+    BLOCKS_DIR = G.BLOCKS_DIR
+    if sector: region = sector_to_region(sector)
+    rx,ry,rz = region
+    rx,ry,rz = rx*32, ry*32, rz*32
+    with open(os.path.join(G.game_dir, world, region_to_filename(region)), "rb") as f:
+        #Load every chunk in this region (4x4x4)
+        for cx in xrange(rx, rx+32, 8):
+            for cy in xrange(ry, ry+32, 8):
+                for cz in xrange(rz, rz+32, 8):
+                    #Now load every block in this chunk (8x8x8)
+                    fstr = f.read(1024)
+                    if fstr != null1024:
+                        fpos = 0
+                        for x in xrange(cx, cx+8):
+                            for y in xrange(cy, cy+8):
+                                for z in xrange(cz, cz+8):
+                                    read = fstr[fpos:fpos+2]
+                                    fpos += 2
+                                    if read != null2:
+                                        position = x,y,z
+                                        blocks[position] = BLOCKS_DIR[structuchar2.unpack(read)]
+                                        sectors[(x/SECTOR_SIZE, y/SECTOR_SIZE, z/SECTOR_SIZE)].append(position)
+
 @performance_info
 def open_world(gamecontroller, game_dir, world=None):
     if world is None: world = "world"
@@ -79,8 +152,11 @@ def open_world(gamecontroller, game_dir, world=None):
     if loaded_save[0] == 3: #Version 3
         if isinstance(loaded_save[1], Player): gamecontroller.player = loaded_save[1]
         if isinstance(loaded_save[2], float): gamecontroller.time_of_day = loaded_save[2]
+
     #blocks and sectors (window.model and window.model.sectors)
-    if G.SAVE_MODE == G.FLATFILE_SAVE_MODE:
+    if G.SAVE_MODE == G.REGION_SAVE_MODE:
+        pass #Sectors are loaded by world._show_sector
+    elif G.SAVE_MODE == G.FLATFILE_SAVE_MODE:
         sectors = gamecontroller.model.sectors
         blocks = gamecontroller.model
         SECTOR_SIZE = G.SECTOR_SIZE

world.py

@@ -105,6 +105,8 @@ def __init__(self):
         self.transparency_batch = pyglet.graphics.Batch()
         self.group = TextureGroup(os.path.join('resources', 'textures', 'texture.png'))
 
+        import savingsystem #This module doesn't like being imported at modulescope
+        self.savingsystem = savingsystem
         self.shown = {}
         self._shown = {}
         self.sectors = defaultdict(list)
@@ -270,7 +272,23 @@ def show_sector(self, sector, immediate=False):
             self.enqueue(self._show_sector, sector, urgent=True)
 
     def _show_sector(self, sector):
-        for position in self.sectors.get(sector, ()):
+        if G.SAVE_MODE == G.REGION_SAVE_MODE and not sector in self.sectors:
+            #The sector is not in memory, load or create it
+            if self.savingsystem.sector_exists(sector):
+                #If its on disk, load it
+                self.savingsystem.load_region(self, sector=sector)
+            else:
+                #The sector doesn't exist yet, generate it!
+                #self.generate_region(key) #<-- TODO
+
+                #Temporary region generation function to show that the world grows
+                cx, cy, cz = self.savingsystem.sector_to_blockpos(sector)
+                rx, ry, rz = cx/32*32, cy/32*32, cz/32*32 #
+                for x in xrange(rx, rx+32):
+                    for z in xrange(rz, rz+32):
+                        self.init_block((x, ry, z), grass_block) #Flat layer of grass at the base of the region
+
+        for position in self.sectors[sector]:
             if position not in self.shown and self.is_exposed(position):
                 self.show_block(position)