Starfield CELL parsed for binary overlays

Mutagen-Modding · Dec 6, 2023 · a081e8f · a081e8f
1 parent 2388dcd
commit a081e8f
Show file tree

Hide file tree

Showing 10 changed files with 339 additions and 78 deletions.
diff --git a/Mutagen.Bethesda.Starfield/Records/Common Subrecords/NavmeshGeometry.xml b/Mutagen.Bethesda.Starfield/Records/Common Subrecords/NavmeshGeometry.xml
@@ -16,7 +16,7 @@
       <P2Float name="GridMaxDistance" />
       <P3Float name="GridMin" />
       <P3Float name="GridMax" />
-      <RefDirect name="GridArrays" refName="NavmeshGridArray" />
+      <RefList name="GridArrays" refName="NavmeshGridArray" />
       <Break />
       <Int32 name="Unknown3" />
     </Fields>

diff --git a/Mutagen.Bethesda.Starfield/Records/Common Subrecords/NavmeshGeometry_Generated.cs b/Mutagen.Bethesda.Starfield/Records/Common Subrecords/NavmeshGeometry_Generated.cs
diff --git a/Mutagen.Bethesda.Starfield/Records/Common Subrecords/TraversalReference.cs b/Mutagen.Bethesda.Starfield/Records/Common Subrecords/TraversalReference.cs
@@ -1,6 +1,9 @@
-using Mutagen.Bethesda.Plugins;
+using System.Buffers.Binary;
+using Mutagen.Bethesda.Plugins;
+using Mutagen.Bethesda.Plugins.Binary.Overlay;
 using Mutagen.Bethesda.Plugins.Binary.Streams;
 using Mutagen.Bethesda.Plugins.Binary.Translations;
+using Mutagen.Bethesda.Starfield.Internals;
 using Noggog;
 
 namespace Mutagen.Bethesda.Starfield;
@@ -98,10 +101,72 @@ public static partial void WriteBinaryTraversalCustom(
 
 partial class TraversalReferenceBinaryOverlay
 {
+    private bool HasFormKey => !Enums.HasFlag(BinaryPrimitives.ReadInt32LittleEndian(_structData.Slice(0x28)), 4);
+
     public partial IFormLinkNullableGetter<ITraversalGetter> GetTraversalCustom(int location)
     {
-        throw new NotImplementedException();
+        if (HasFormKey)
+        {
+            return new FormLinkNullable<ITraversalGetter>(
+                FormKey.Factory(_package.MetaData.MasterReferences!, BinaryPrimitives.ReadUInt32LittleEndian(
+                    _structData.Slice(0x2C))));
+        }
+        else
+        {
+            return FormLinkNullable<ITraversalGetter>.Null;
+        }
+    }
+
+    public ReadOnlyMemorySlice<byte> Unknown => _structData.Slice(HasFormKey ? 0x30 : 0x2C, 8);
+
+    public static IReadOnlyList<ITraversalReferenceGetter> Factory(
+        OverlayStream stream, BinaryOverlayFactoryPackage package,
+        long finalPos, int offset, PreviousParse lastParsed)
+    {
+        stream.ReadSubrecordHeader(RecordTypes.XTV2);
+        return BinaryOverlayList.FactoryByArray(
+            mem: stream.RemainingMemory,
+            package: package,
+            getter: (s, p) =>
+            {
+                return TraversalReferenceFactory(s, p);
+            },
+            locs: GetLocs(stream, package, finalPos, offset, lastParsed));
+    }
+
+    public static IReadOnlyList<int> GetLocs(
+        OverlayStream stream, BinaryOverlayFactoryPackage package,
+        long finalPos, int offset, PreviousParse lastParsed)
+    {
+        List<int> locs = new();
+        var startingPos = stream.Position;
+        while (stream.Position < finalPos)
+        {
+            try
+            {
+                var itemPos = stream.Position;
+                var bytes = stream.ReadMemory(0x10);
+                if (bytes.All(b => b == 0)) break;
+                locs.Add(itemPos - startingPos);
+                stream.Position += 0x18;
+                var flags = stream.ReadInt32();
+                if (Enums.HasFlag(flags, 4))
+                {
+                    stream.Position += 8;
+                }
+                else
+                {
+                    stream.Position += 12;
+                }
+            }
+            catch (ArgumentOutOfRangeException)
+            {
+                // Expected to occur within Starfield.esm, due to fluff bytes of unknown length
+                stream.Position += checked((int)stream.Remaining);
+                break;
+            }
+        }
+
+        return locs;
     }
-
-    public ReadOnlyMemorySlice<byte> Unknown { get; }
 }
diff --git a/Mutagen.Bethesda.Starfield/Records/Common Subrecords/VolumesComponentItem.cs b/Mutagen.Bethesda.Starfield/Records/Common Subrecords/VolumesComponentItem.cs
@@ -59,16 +59,31 @@ public static partial void FillBinaryEnderCustom(
 
 partial class VolumesComponentItemBinaryOverlay
 {
+    partial void CustomEnderEndPos()
+    {
+        var val = BinaryPrimitives.ReadInt32LittleEndian(_structData);
+        EnderEndingPos = 0x50;
+        EnderEndingPos += val switch
+        {
+            1 => 4,
+            3 => 8,
+            5 => 12,
+            _ => throw new NotImplementedException()
+        };
+    }
+
     public partial IAVolumesUnknownEnderGetter GetEnderCustom(int location)
     {
-        return BinaryPrimitives.ReadInt32LittleEndian(_structData) switch
+        var val = BinaryPrimitives.ReadInt32LittleEndian(_structData);
+        return val switch
         {
             1 => VolumesUnknownEnderSingleBinaryOverlay.VolumesUnknownEnderSingleFactory(_structData.Slice(location),
                 _package),
             3 => VolumesUnknownEnderDoubleBinaryOverlay.VolumesUnknownEnderDoubleFactory(_structData.Slice(location),
                 _package),
             5 => VolumesUnknownEnderTrioBinaryOverlay.VolumesUnknownEnderTrioFactory(_structData.Slice(location),
                 _package),
+            _ => throw new NotImplementedException()
         };
     }
 }
diff --git a/Mutagen.Bethesda.Starfield/Records/Major Records/Cell.cs b/Mutagen.Bethesda.Starfield/Records/Major Records/Cell.cs
@@ -359,7 +359,7 @@ partial class CellBinaryOverlay
 
     private ReadOnlyMemorySlice<byte>? _grupData;
 
-    public IReadOnlyList<ITraversalReferenceGetter>? Traversals => throw new NotImplementedException();
+    public IReadOnlyList<ITraversalReferenceGetter>? Traversals { get; private set; }
     public IReadOnlyList<INavigationMeshGetter> NavigationMeshes { get; private set; } = Array.Empty<INavigationMeshGetter>();
 
     public int UnknownGroupData => _grupData.HasValue ? BinaryPrimitives.ReadInt32LittleEndian(_grupData.Value.Slice(20)) : default;
@@ -572,4 +572,9 @@ IPlacedGetter TypicalGetter(
             }
         }
     }
+
+    partial void TraversalsCustomParse(OverlayStream stream, long finalPos, int offset, RecordType type, PreviousParse lastParsed)
+    {
+        Traversals = TraversalReferenceBinaryOverlay.Factory(stream, _package, finalPos, offset, lastParsed);
+    }
 }
diff --git a/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject.cs b/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject.cs
@@ -1,4 +1,10 @@
-namespace Mutagen.Bethesda.Starfield;
+using Mutagen.Bethesda.Plugins;
+using Mutagen.Bethesda.Plugins.Binary.Overlay;
+using Mutagen.Bethesda.Plugins.Binary.Streams;
+using Mutagen.Bethesda.Plugins.Binary.Translations;
+using Mutagen.Bethesda.Starfield.Internals;
+
+namespace Mutagen.Bethesda.Starfield;
 
 public partial class PlacedObject
 {
@@ -155,4 +161,62 @@ public enum ItemMajorFlag : uint
         NoRespawn = 0x4000_0000,
         Multibound = 0x8000_0000,
     }
+}
+
+partial class PlacedObjectBinaryWriteTranslation
+{
+    public static partial void WriteBinaryTraversalsCustom(
+        MutagenWriter writer,
+        IPlacedObjectGetter item)
+    {
+        Mutagen.Bethesda.Plugins.Binary.Translations.ListBinaryTranslation<ITraversalReferenceGetter>.Instance.Write(
+            writer: writer,
+            items: item.Traversals,
+            recordType: RecordTypes.XTV2,
+            overflowRecord: RecordTypes.XXXX,
+            transl: (MutagenWriter subWriter, ITraversalReferenceGetter subItem, TypedWriteParams conv) =>
+            {
+                var Item = subItem;
+                ((TraversalReferenceBinaryWriteTranslation)((IBinaryItem)Item).BinaryWriteTranslator).Write(
+                    item: Item,
+                    writer: subWriter,
+                    translationParams: conv);
+            });
+    }
+}
+
+partial class PlacedObjectBinaryCreateTranslation
+{
+    public static partial void FillBinaryTraversalsCustom(
+        MutagenFrame frame,
+        IPlacedObjectInternal item,
+        PreviousParse lastParsed)
+    {
+        var sub = frame.ReadSubrecordHeader();
+        int len;
+        if (lastParsed.LengthOverride.HasValue)
+        {
+            len = lastParsed.LengthOverride.Value;
+        }
+        else
+        {
+            len = sub.ContentLength;
+        }
+        item.Traversals = TraversalReferenceBinaryCreateTranslation.Parse(frame.SpawnWithLength(len));
+    }
+}
+
+partial class PlacedObjectBinaryOverlay
+{
+    public IReadOnlyList<ITraversalReferenceGetter>? Traversals { get; private set; }
+
+    partial void TraversalsCustomParse(
+        OverlayStream stream,
+        long finalPos,
+        int offset,
+        RecordType type,
+        PreviousParse lastParsed)
+    {
+        Traversals = TraversalReferenceBinaryOverlay.Factory(stream, _package, finalPos, offset, lastParsed);
+    }
 }
diff --git a/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject.xml b/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject.xml
@@ -82,7 +82,7 @@
       <Percent name="HealthPercent" recordType="XHLT" />
       <FormLink name="TimeOfDay" refName="TimeOfDayData" recordType="TODD" />
       <RefDirect name="EnableParent" refName="EnableParent" />
-      <RefList name="Traversals" refName="TraversalReference" recordType="XTV2" />
+      <RefList name="Traversals" refName="TraversalReference" recordType="XTV2" binary="Custom" />
       <RefDirect name="NavigationDoorLink" refName="NavigationDoorLink" />
       <Bool name="IsActivationPoint" boolAsMarker="XATP" />
       <Float name="Scale" recordType="XSCL" />

diff --git a/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject_Generated.cs b/Mutagen.Bethesda.Starfield/Records/Major Records/PlacedObject_Generated.cs
@@ -8374,18 +8374,9 @@ public static void WriteRecordTypes(
                     writer: writer,
                     translationParams: translationParams);
             }
-            Mutagen.Bethesda.Plugins.Binary.Translations.ListBinaryTranslation<ITraversalReferenceGetter>.Instance.Write(
+            PlacedObjectBinaryWriteTranslation.WriteBinaryTraversals(
                 writer: writer,
-                items: item.Traversals,
-                recordType: translationParams.ConvertToCustom(RecordTypes.XTV2),
-                transl: (MutagenWriter subWriter, ITraversalReferenceGetter subItem, TypedWriteParams conv) =>
-                {
-                    var Item = subItem;
-                    ((TraversalReferenceBinaryWriteTranslation)((IBinaryItem)Item).BinaryWriteTranslator).Write(
-                        item: Item,
-                        writer: subWriter,
-                        translationParams: conv);
-                });
+                item: item);
             if (item.NavigationDoorLink is {} NavigationDoorLinkItem)
             {
                 ((NavigationDoorLinkBinaryWriteTranslation)((IBinaryItem)NavigationDoorLinkItem).BinaryWriteTranslator).Write(
@@ -8421,6 +8412,19 @@ public static void WriteRecordTypes(
                 binaryType: StringBinaryType.NullTerminate);
         }
 
+        public static partial void WriteBinaryTraversalsCustom(
+            MutagenWriter writer,
+            IPlacedObjectGetter item);
+
+        public static void WriteBinaryTraversals(
+            MutagenWriter writer,
+            IPlacedObjectGetter item)
+        {
+            WriteBinaryTraversalsCustom(
+                writer: writer,
+                item: item);
+        }
+
         public void Write(
             MutagenWriter writer,
             IPlacedObjectGetter item,
@@ -8939,12 +8943,10 @@ public static ParseResult FillBinaryRecordTypes(
                 }
                 case RecordTypeInts.XTV2:
                 {
-                    frame.Position += frame.MetaData.Constants.SubConstants.HeaderLength;
-                    item.Traversals = 
-                        Mutagen.Bethesda.Plugins.Binary.Translations.ListBinaryTranslation<TraversalReference>.Instance.Parse(
-                            reader: frame.SpawnWithLength(contentLength),
-                            transl: TraversalReference.TryCreateFromBinary)
-                        .CastExtendedList<TraversalReference>();
+                    PlacedObjectBinaryCreateTranslation.FillBinaryTraversalsCustom(
+                        frame: frame.SpawnWithLength(frame.MetaData.Constants.SubConstants.HeaderLength + contentLength),
+                        item: item,
+                        lastParsed: lastParsed);
                     return (int)PlacedObject_FieldIndex.Traversals;
                 }
                 case RecordTypeInts.XNDP:
@@ -9003,6 +9005,11 @@ public static ParseResult FillBinaryRecordTypes(
             }
         }
 
+        public static partial void FillBinaryTraversalsCustom(
+            MutagenFrame frame,
+            IPlacedObjectInternal item,
+            PreviousParse lastParsed);
+
     }
 
 }
@@ -9272,7 +9279,14 @@ void IBinaryItem.WriteToBinary(
         private RangeInt32? _EnableParentLocation;
         public IEnableParentGetter? EnableParent => _EnableParentLocation.HasValue ? EnableParentBinaryOverlay.EnableParentFactory(_recordData.Slice(_EnableParentLocation!.Value.Min), _package) : default;
         #endregion
-        public IReadOnlyList<ITraversalReferenceGetter>? Traversals { get; private set; }
+        #region Traversals
+        partial void TraversalsCustomParse(
+            OverlayStream stream,
+            long finalPos,
+            int offset,
+            RecordType type,
+            PreviousParse lastParsed);
+        #endregion
         #region NavigationDoorLink
         private RangeInt32? _NavigationDoorLinkLocation;
         public INavigationDoorLinkGetter? NavigationDoorLink => _NavigationDoorLinkLocation.HasValue ? NavigationDoorLinkBinaryOverlay.NavigationDoorLinkFactory(_recordData.Slice(_NavigationDoorLinkLocation!.Value.Min), _package) : default;
@@ -9802,13 +9816,12 @@ public override ParseResult FillRecordType(
                 }
                 case RecordTypeInts.XTV2:
                 {
-                    var subMeta = stream.ReadSubrecordHeader();
-                    var subLen = finalPos - stream.Position;
-                    this.Traversals = BinaryOverlayList.FactoryByLazyParse<ITraversalReferenceGetter>(
-                        mem: stream.RemainingMemory.Slice(0, subLen),
-                        package: _package,
-                        getter: (s, p) => TraversalReferenceBinaryOverlay.TraversalReferenceFactory(s, p));
-                    stream.Position += subLen;
+                    TraversalsCustomParse(
+                        stream: stream,
+                        finalPos: finalPos,
+                        offset: offset,
+                        type: type,
+                        lastParsed: lastParsed);
                     return (int)PlacedObject_FieldIndex.Traversals;
                 }
                 case RecordTypeInts.XNDP:

diff --git a/Mutagen.Bethesda.Tests/Passthrough Tests/PassthroughTest.cs b/Mutagen.Bethesda.Tests/Passthrough Tests/PassthroughTest.cs
@@ -399,10 +399,19 @@ public Test BinaryPassthroughTest()
                     o.OnNext(FilePath.ToString());
                     using (var wrapper = await ImportBinaryOverlay(trimmedPath, StringsParams))
                     {
-                        doStrings = wrapper.UsingLocalization;
-                        var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(GameRelease, wrapper.ModKey, strsWriteDir, MutagenEncoding.Default) : null);
-                        wrapper.WriteToBinary(binaryOverlayPath, writeParam);
-                        writeParam.StringsWriter?.Dispose();
+                        try
+                        {
+
+                            doStrings = wrapper.UsingLocalization;
+                            var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(GameRelease, wrapper.ModKey, strsWriteDir, MutagenEncoding.Default) : null);
+                            wrapper.WriteToBinary(binaryOverlayPath, writeParam);
+                            writeParam.StringsWriter?.Dispose();
+                        }
+                        catch (Exception e)
+                        {
+                            Console.WriteLine(e);
+                            throw;
+                        }
                     }
 
                     using var stream = new MutagenBinaryReadStream(processedPath, GameRelease);

diff --git a/Mutagen.Bethesda.Tests/Processing/StarfieldProcessor.cs b/Mutagen.Bethesda.Tests/Processing/StarfieldProcessor.cs
@@ -427,7 +427,6 @@ private void ProcessXTV2(MajorRecordFrame majorFrame, long fileOffset)
                 if (remainingLen < ushort.MaxValue && xtv2.LengthOverrideRecordLocation.HasValue)
                 {
                     RemoveOverflowRecord(majorFrame, xtv2, fileOffset, checked((uint)remainingLen));
-                    xtv2 = xtv2.WithoutOverflow();
                     cutLen += 10;
                     ProcessLengths(majorFrame, -cutLen, fileOffset);
                 }