improved auto-documentation for simple programmatic array lengths

library/context.py

@@ -43,3 +43,45 @@ def __init__(self, result: bytes = b'', name: str = '', data=None, block=None, p
         super().__init__(name=name, data=data, block=block, parent=parent)
         self.result = result
         self.write_start_offset = len(result)
+
+
+class DocumentationCtxData:
+    def __init__(self, label):
+        self.label = label
+
+    def __str__(self):
+        return self.label
+
+    def __add__(self, other):
+        a = str(self)
+        b = str(other)
+        return DocumentationCtxData(f'{a}+{b}')
+
+    def __radd__(self, other):
+        a = str(other)
+        b = str(self)
+        return DocumentationCtxData(f'{a}+{b}')
+
+    def __mul__(self, other):
+        a = str(self)
+        b = str(other)
+        if '+' in a or '-' in a:
+            a = f'({a})'
+        if '+' in b or '-' in b:
+            b = f'({b})'
+        return DocumentationCtxData(f'{a}*{b}')
+
+    def __rmul__(self, other):
+        a = str(other)
+        b = str(self)
+        if '+' in a or '-' in a:
+            a = f'({a})'
+        if '+' in b or '-' in b:
+            b = f'({b})'
+        return DocumentationCtxData(f'{a}*{b}')
+
+
+class DocumentationContext(BaseContext):
+
+    def data(self, local_path: str):
+        return DocumentationCtxData(local_path.replace('../', '^'))

library/read_blocks/array.py

@@ -3,7 +3,7 @@
 from math import ceil
 from typing import Dict, Tuple, Any
 
-from library.context import ReadContext, WriteContext
+from library.context import ReadContext, WriteContext, DocumentationContext
 from library.exceptions import EndOfBufferException
 from library.read_blocks.basic import DataBlock, DataBlockWithChildren
 from library.read_blocks.numbers import IntegerBlock
@@ -31,7 +31,10 @@ def length_doc_str(self):
             (_, doc_str) = self._length
             return doc_str
         if callable(self._length):
-            return "custom_func"
+            try:
+                return str(self._length(DocumentationContext()))
+            except:
+                return 'custom_func'
         return str(self._length)
 
     # For auto-generated documentation only
@@ -159,7 +162,10 @@ def length_doc_str(self):
             (_, doc_str) = self._length
             return doc_str
         if callable(self._length):
-            return "custom_func"
+            try:
+                return str(self._length(DocumentationContext()))
+            except:
+                return 'custom_func'
         return str(self._length)
 
     # For auto-generated documentation only

library/read_blocks/basic.py

@@ -2,7 +2,7 @@
 from io import BufferedReader, BytesIO, SEEK_CUR
 from typing import Dict, Any, Tuple, Literal
 
-from library.context import ReadContext, WriteContext
+from library.context import ReadContext, WriteContext, DocumentationContext
 from library.exceptions import DataIntegrityException, BlockDefinitionException, EndOfBufferException
 from library.utils import represent_value_as_str
 
@@ -97,7 +97,10 @@ def size_doc_str(self):
             (_, doc_str) = self._length
             return doc_str
         if callable(self._length):
-            return "custom_func"
+            try:
+                return str(self._length(DocumentationContext()))
+            except:
+                return 'custom_func'
         else:
             return str(self._length)
 

library/read_blocks/smart_fields.py

@@ -1,7 +1,7 @@
 from io import BufferedReader, BytesIO
 from typing import List, Dict, Tuple, Any
 
-from library.context import ReadContext, WriteContext
+from library.context import ReadContext, WriteContext, DocumentationContext
 from library.exceptions import DataIntegrityException
 from library.read_blocks.basic import DataBlock, SkipBlock, BytesBlock
 from library.utils.id import join_id
@@ -20,7 +20,10 @@ def schema(self) -> Dict:
         if isinstance(choice_index_doc, tuple):
             (_, choice_index_doc) = choice_index_doc
         if callable(choice_index_doc):
-            choice_index_doc = "custom_func"
+            try:
+                choice_index_doc = str(choice_index_doc(DocumentationContext()))
+            except:
+                choice_index_doc = 'custom_func'
         return {
             **super().schema,
             'possible_resource_schemas': [block.schema for block in self.possible_blocks],

library/read_blocks/strings.py

@@ -1,7 +1,7 @@
 from io import BufferedReader, BytesIO
 from typing import Dict
 
-from library.context import ReadContext, WriteContext
+from library.context import ReadContext, WriteContext, DocumentationContext
 from library.exceptions import EndOfBufferException
 from library.read_blocks.basic import DataBlock
 
@@ -30,7 +30,10 @@ def size_doc_str(self):
             (_, doc_str) = self._length
             return doc_str
         if callable(self._length):
-            return "custom_func"
+            try:
+                return str(self._length(DocumentationContext()))
+            except:
+                return 'custom_func'
         return str(self._length)
 
     def resolve_length(self, ctx):

resources/NFS2.md

@@ -126,7 +126,7 @@ Did not find what you need or some given data is wrong? Please submit an
 | 8 | **unk** | 4 | Bytes | Unknown purpose |
 | 12 | **x** | 2 | 2-bytes unsigned integer (little endian) | X coordinate of bitmap position on screen. Used for menu/dash sprites |
 | 14 | **y** | 2 | 2-bytes unsigned integer (little endian) | Y coordinate of bitmap position on screen. Used for menu/dash sprites |
-| 16 | **bitmap** | height\*ceil((width)\*4/8) | Array of `height` items<br/>Item size: ceil((width)\*4/8) bytes<br/>Item type: Array of `width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
+| 16 | **bitmap** | height\*ceil((^width)\*4/8) | Array of `height` items<br/>Item size: ceil((^width)\*4/8) bytes<br/>Item type: Array of `^width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
 ### **Bitmap8Bit** ###
 #### **Size**: 16..? bytes ####
 #### **Description**: 8bit bitmap can be serialized to image only with palette. Basically, for every pixel it uses 8-bit index of color in assigned palette. The tricky part is to determine how the game understands which palette to use. In most cases, if bitmap has embedded palette, it should be used, EXCEPT Autumn Valley fence texture: there embedded palette should be ignored. In all other cases it is tricky even more: it uses !pal or !PAL palette from own SHPI archive, if it is WWWW archive, palette can be in a different SHPI before this one. In CONTROL directory most of QFS files use !pal even from different QFS file! It is a mystery how to reliably pick palette ####

resources/NFS2_SE.md

@@ -122,7 +122,7 @@ Did not find what you need or some given data is wrong? Please submit an
 | 8 | **unk** | 4 | Bytes | Unknown purpose |
 | 12 | **x** | 2 | 2-bytes unsigned integer (little endian) | X coordinate of bitmap position on screen. Used for menu/dash sprites |
 | 14 | **y** | 2 | 2-bytes unsigned integer (little endian) | Y coordinate of bitmap position on screen. Used for menu/dash sprites |
-| 16 | **bitmap** | height\*ceil((width)\*4/8) | Array of `height` items<br/>Item size: ceil((width)\*4/8) bytes<br/>Item type: Array of `width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
+| 16 | **bitmap** | height\*ceil((^width)\*4/8) | Array of `height` items<br/>Item size: ceil((^width)\*4/8) bytes<br/>Item type: Array of `^width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
 ### **Bitmap8Bit** ###
 #### **Size**: 16..? bytes ####
 #### **Description**: 8bit bitmap can be serialized to image only with palette. Basically, for every pixel it uses 8-bit index of color in assigned palette. The tricky part is to determine how the game understands which palette to use. In most cases, if bitmap has embedded palette, it should be used, EXCEPT Autumn Valley fence texture: there embedded palette should be ignored. In all other cases it is tricky even more: it uses !pal or !PAL palette from own SHPI archive, if it is WWWW archive, palette can be in a different SHPI before this one. In CONTROL directory most of QFS files use !pal even from different QFS file! It is a mystery how to reliably pick palette ####

resources/NFS3.md

@@ -54,7 +54,7 @@ Did not find what you need or some given data is wrong? Please submit an
 | 8 | **unk** | 4 | Bytes | Unknown purpose |
 | 12 | **x** | 2 | 2-bytes unsigned integer (little endian) | X coordinate of bitmap position on screen. Used for menu/dash sprites |
 | 14 | **y** | 2 | 2-bytes unsigned integer (little endian) | Y coordinate of bitmap position on screen. Used for menu/dash sprites |
-| 16 | **bitmap** | height\*ceil((width)\*4/8) | Array of `height` items<br/>Item size: ceil((width)\*4/8) bytes<br/>Item type: Array of `width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
+| 16 | **bitmap** | height\*ceil((^width)\*4/8) | Array of `height` items<br/>Item size: ceil((^width)\*4/8) bytes<br/>Item type: Array of `^width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
 ### **Bitmap8Bit** ###
 #### **Size**: 16..? bytes ####
 #### **Description**: 8bit bitmap can be serialized to image only with palette. Basically, for every pixel it uses 8-bit index of color in assigned palette. The tricky part is to determine how the game understands which palette to use. In most cases, if bitmap has embedded palette, it should be used, EXCEPT Autumn Valley fence texture: there embedded palette should be ignored. In all other cases it is tricky even more: it uses !pal or !PAL palette from own SHPI archive, if it is WWWW archive, palette can be in a different SHPI before this one. In CONTROL directory most of QFS files use !pal even from different QFS file! It is a mystery how to reliably pick palette ####

resources/NFS4.md

@@ -54,7 +54,7 @@ Did not find what you need or some given data is wrong? Please submit an
 | 8 | **unk** | 4 | Bytes | Unknown purpose |
 | 12 | **x** | 2 | 2-bytes unsigned integer (little endian) | X coordinate of bitmap position on screen. Used for menu/dash sprites |
 | 14 | **y** | 2 | 2-bytes unsigned integer (little endian) | Y coordinate of bitmap position on screen. Used for menu/dash sprites |
-| 16 | **bitmap** | height\*ceil((width)\*4/8) | Array of `height` items<br/>Item size: ceil((width)\*4/8) bytes<br/>Item type: Array of `width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
+| 16 | **bitmap** | height\*ceil((^width)\*4/8) | Array of `height` items<br/>Item size: ceil((^width)\*4/8) bytes<br/>Item type: Array of `^width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
 ### **Bitmap8Bit** ###
 #### **Size**: 16..? bytes ####
 #### **Description**: 8bit bitmap can be serialized to image only with palette. Basically, for every pixel it uses 8-bit index of color in assigned palette. The tricky part is to determine how the game understands which palette to use. In most cases, if bitmap has embedded palette, it should be used, EXCEPT Autumn Valley fence texture: there embedded palette should be ignored. In all other cases it is tricky even more: it uses !pal or !PAL palette from own SHPI archive, if it is WWWW archive, palette can be in a different SHPI before this one. In CONTROL directory most of QFS files use !pal even from different QFS file! It is a mystery how to reliably pick palette ####

resources/TNFS_SE.md

@@ -367,7 +367,7 @@ Did not find what you need or some given data is wrong? Please submit an
 | 8 | **unk** | 4 | Bytes | Unknown purpose |
 | 12 | **x** | 2 | 2-bytes unsigned integer (little endian) | X coordinate of bitmap position on screen. Used for menu/dash sprites |
 | 14 | **y** | 2 | 2-bytes unsigned integer (little endian) | Y coordinate of bitmap position on screen. Used for menu/dash sprites |
-| 16 | **bitmap** | height\*ceil((width)\*4/8) | Array of `height` items<br/>Item size: ceil((width)\*4/8) bytes<br/>Item type: Array of `width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
+| 16 | **bitmap** | height\*ceil((^width)\*4/8) | Array of `height` items<br/>Item size: ceil((^width)\*4/8) bytes<br/>Item type: Array of `^width` sub-byte numbers. Each number consists of 4 bits | Font atlas bitmap data, array of bitmap rows |
 ### **Bitmap8Bit** ###
 #### **Size**: 16..? bytes ####
 #### **Description**: 8bit bitmap can be serialized to image only with palette. Basically, for every pixel it uses 8-bit index of color in assigned palette. The tricky part is to determine how the game understands which palette to use. In most cases, if bitmap has embedded palette, it should be used, EXCEPT Autumn Valley fence texture: there embedded palette should be ignored. In all other cases it is tricky even more: it uses !pal or !PAL palette from own SHPI archive, if it is WWWW archive, palette can be in a different SHPI before this one. In CONTROL directory most of QFS files use !pal even from different QFS file! It is a mystery how to reliably pick palette ####

resources/eac/archives.py

@@ -196,7 +196,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
                                                       inline_description='8-bytes record, first 4 bytes is a UTF-8 '
                                                                          'string, last 4 bytes is an unsigned integer '
                                                                          '(little-endian)'),
-                                  length=(lambda ctx: ctx.data('num_items'), 'num_items')),
+                                  length=lambda ctx: ctx.data('num_items')),
                        {'description': 'An array of items, each of them represents name of SHPI item (image or palette)'
                                        ' and offset to item data in file, relatively to SHPI block start (where '
                                        'resource id string is presented). Names are not always unique'})
@@ -286,7 +286,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
                      {'description': 'An amount of items',
                       'programmatic_value': lambda ctx: len(ctx.data('items_descr'))})
         items_descr = (ArrayBlock(child=IntegerBlock(length=4),
-                                  length=(lambda ctx: ctx.data('num_items'), 'num_items')),
+                                  length=lambda ctx: ctx.data('num_items')),
                        {'description': 'An array of offsets to items data in file, relatively to wwww block start '
                                        '(where resource id string is presented)'})
         children = (ArrayBlock(length=(0, 'num_items'), child=None),
@@ -355,7 +355,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
                       'programmatic_value': lambda ctx: len(ctx.data('items_descr'))})
         unk0 = (IntegerBlock(length=4),
                 {'is_unknown': True})
-        items_descr = ArrayBlock(length=(lambda ctx: ctx.data('num_items'), 'num_items'),
+        items_descr = ArrayBlock(length=lambda ctx: ctx.data('num_items'),
                                  child=CompoundBlock(fields=[('offset', IntegerBlock(length=4, byte_order='big'), {}),
                                                              ('length', IntegerBlock(length=4, byte_order='big'), {}),
                                                              ('name', NullTerminatedUTF8Block(length=8), {})]))

resources/eac/bitmaps.py

@@ -35,7 +35,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
         bitmap = (ArrayBlock(child=Color16Bit0565Block(simplified=True),
-                             length=(lambda ctx: ctx.data('width') * ctx.data('height'), 'width*height')),
+                             length=lambda ctx: ctx.data('width') * ctx.data('height')),
                   {'description': 'Colors of bitmap pixels'})
 
 
@@ -65,9 +65,9 @@ class Fields(DeclarativeCompoundBlock.Fields):
              {'description': 'X coordinate of bitmap position on screen. Used for menu/dash sprites'})
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
-        bitmap = (ArrayBlock(length=(lambda ctx: ctx.data('height'), 'height'),
+        bitmap = (ArrayBlock(length=lambda ctx: ctx.data('height'),
                              child=SubByteArrayBlock(bits_per_value=4,
-                                                     length=(lambda ctx: ctx.data('../width'), 'width'),
+                                                     length=lambda ctx: ctx.data('../width'),
                                                      value_deserialize_func=lambda x: 0xFFFFFF00
                                                                                       | transform_bitness(x, 4),
                                                      value_serialize_func=lambda x: (x & 0xFF) >> 4)),
@@ -102,15 +102,15 @@ class Fields(DeclarativeCompoundBlock.Fields):
         unk = (IntegerBlock(length=2),
                {'is_unknown': True})
         pivot_y = (IntegerBlock(length=2),
-                       {'description': 'For "horz" bitmap in TNFS FAM files: Y coordinate of the horizon line on '
-                                       'the image. Higher value = image as horizon will be put higher on the screen. '
-                                       'Seems to affect only open tracks'})
+                   {'description': 'For "horz" bitmap in TNFS FAM files: Y coordinate of the horizon line on '
+                                   'the image. Higher value = image as horizon will be put higher on the screen. '
+                                   'Seems to affect only open tracks'})
         x = (IntegerBlock(length=2),
              {'description': 'X coordinate of bitmap position on screen. Used for menu/dash sprites'})
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
         bitmap = (ArrayBlock(child=IntegerBlock(length=1),
-                             length=(lambda ctx: ctx.data('width') * ctx.data('height'), 'width*height')),
+                             length=lambda ctx: ctx.data('width') * ctx.data('height')),
                   {'description': 'Color indexes of bitmap pixels. The actual colors are '
                                   'in assigned to this bitmap palette'})
 
@@ -132,7 +132,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
         bitmap = (ArrayBlock(child=Color32BitBlock(),
-                             length=(lambda ctx: ctx.data('width') * ctx.data('height'), 'width*height')),
+                             length=lambda ctx: ctx.data('width') * ctx.data('height')),
                   {'description': 'Colors of bitmap pixels'})
 
 
@@ -153,7 +153,7 @@ class Fields(DeclarativeCompoundBlock.Fields):
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
         bitmap = (ArrayBlock(child=Color16Bit1555Block(),
-                             length=(lambda ctx: ctx.data('width') * ctx.data('height'), 'width*height')),
+                             length=lambda ctx: ctx.data('width') * ctx.data('height')),
                   {'description': 'Colors of bitmap pixels'})
 
 
@@ -174,5 +174,5 @@ class Fields(DeclarativeCompoundBlock.Fields):
         y = (IntegerBlock(length=2),
              {'description': 'Y coordinate of bitmap position on screen. Used for menu/dash sprites'})
         bitmap = (ArrayBlock(child=Color24BitLittleEndianField(),
-                             length=(lambda ctx: ctx.data('width') * ctx.data('height'), 'width*height')),
+                             length=lambda ctx: ctx.data('width') * ctx.data('height')),
                   {'description': 'Colors of bitmap pixels'})