From 043203e238c7b35eb397bca19bc2ee92aef33549 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:08:27 +0200 Subject: [PATCH 01/14] [ModSupport] Added WIP mod format stuff --- FrostyModSupport/FrostyModExecutor.cs | 255 ++++++++++++++++++ .../Interfaces/IResourceContainer.cs | 8 + FrostyModSupport/Mod/FrostyMod.cs | 209 ++++++++++++++ FrostyModSupport/Mod/FrostyModCollection.cs | 19 ++ FrostyModSupport/Mod/FrostyModDetails.cs | 51 ++++ .../Mod/Resources/BaseModResource.cs | 147 ++++++++++ .../Mod/Resources/BundleModResource.cs | 34 +++ .../Mod/Resources/ChunkModResource.cs | 86 ++++++ .../Mod/Resources/EbxModResource.cs | 22 ++ .../Mod/Resources/EmbeddedModResource.cs | 20 ++ .../Mod/Resources/FsFileModResource.cs | 13 + .../Mod/Resources/ModResourceType.cs | 42 +++ .../Mod/Resources/ResModResource.cs | 42 +++ FrostyModSupport/ModEntries/ChunkModEntry.cs | 6 + FrostyModSupport/ModEntries/EbxModEntry.cs | 12 + FrostyModSupport/ModEntries/ResModEntry.cs | 6 + FrostyModSupport/ModInfo.cs | 30 +++ FrostyModSupport/ModInfos/BundleModAction.cs | 6 +- .../ModInfos/SuperBundleModAction.cs | 4 +- 19 files changed, 1007 insertions(+), 5 deletions(-) create mode 100644 FrostyModSupport/FrostyModExecutor.cs create mode 100644 FrostyModSupport/Interfaces/IResourceContainer.cs create mode 100644 FrostyModSupport/Mod/FrostyMod.cs create mode 100644 FrostyModSupport/Mod/FrostyModCollection.cs create mode 100644 FrostyModSupport/Mod/FrostyModDetails.cs create mode 100755 FrostyModSupport/Mod/Resources/BaseModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/BundleModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/ChunkModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/EbxModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/EmbeddedModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/FsFileModResource.cs create mode 100755 FrostyModSupport/Mod/Resources/ModResourceType.cs create mode 100755 FrostyModSupport/Mod/Resources/ResModResource.cs create mode 100644 FrostyModSupport/ModEntries/ChunkModEntry.cs create mode 100644 FrostyModSupport/ModEntries/EbxModEntry.cs create mode 100644 FrostyModSupport/ModEntries/ResModEntry.cs create mode 100644 FrostyModSupport/ModInfo.cs diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs new file mode 100644 index 000000000..5f739a5c3 --- /dev/null +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -0,0 +1,255 @@ +using System.Text.Json; +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod; +using Frosty.ModSupport.Mod.Resources; +using Frosty.ModSupport.ModEntries; +using Frosty.ModSupport.ModInfos; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Managers.Entries; +using Frosty.Sdk.Managers.Infos; + +namespace Frosty.ModSupport; + +public class FrostyModExecutor +{ + private Dictionary m_modifiedEbx = new(); + private Dictionary m_modifiedRes = new(); + private Dictionary m_modifiedChunks = new(); + + private Dictionary m_superBundleModInfos = new(); + private Dictionary m_mapping = new(); + + /// + /// Generates a directory containing the modded games data. + /// + /// The name of the directory where the data is stored in the games ModData folder. + /// The full paths of the mods. + public void GenerateMods(string modPackName, params string[] modPaths) + { + string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName); + string patchPath = FileSystemManager.Sources.Count == 1 + ? FileSystemSource.Base.Path + : FileSystemSource.Patch.Path; + + // check if we need to generate new data + string modInfosPath = Path.Combine(modDataPath, patchPath, "mods.json"); + List modInfos = GenerateModInfoList(modPaths); + if (File.Exists(modInfosPath)) + { + List? oldModInfos = JsonSerializer.Deserialize>(File.ReadAllText(modInfosPath)); + if (oldModInfos?.SequenceEqual(modInfos) == true) + { + return; + } + } + + // make sure the managers are initialized + ResourceManager.Initialize(); + AssetManager.Initialize(); + + // create bundle lookup map + GenerateBundleLookup(); + + // process all mods + foreach (string path in modPaths) + { + string extension = Path.GetExtension(path); + if (extension == ".fbmod") + { + FrostyMod mod = FrostyMod.Load(path); + ProcessModResources(mod); + } + else if (extension == ".fbcollection") + { + FrostyModCollection modCollection = FrostyModCollection.Load(path); + ProcessModResources(modCollection); + } + else + { + throw new Exception(); + } + } + } + + private void GenerateBundleLookup() + { + foreach (SuperBundleInfo sb in FileSystemManager.EnumerateSuperBundles()) + { + foreach (SuperBundleInstallChunk sbIc in sb.InstallChunks) + { + foreach (KeyValuePair bundle in sbIc.BundleMapping) + { + m_mapping.Add(bundle.Value.Id, sbIc.Name); + } + } + } + + } + + private void ProcessModResources(IResourceContainer container) + { + foreach (BaseModResource resource in container.Resources) + { + HashSet modifiedBundles = new(); + switch (resource) + { + case BundleModResource: + break; + case EbxModResource: + { + if (resource.IsModified || !m_modifiedEbx.ContainsKey(resource.Name)) + { + if (resource.HasHandler) + { + + } + else + { + if (m_modifiedEbx.TryGetValue(resource.Name, out EbxModEntry? existingEntry)) + { + if (existingEntry.Sha1 == resource.Sha1 /*|| has handler*/) + { + break; + } + + m_modifiedEbx.Remove(resource.Name, out _); + } + + // TODO: create EbxModEntry from resource + + EbxAssetEntry? ebxEntry = AssetManager.GetEbxAssetEntry(resource.Name); + + if (resource.ResourceIndex == -1) + { + // only add asset to bundles, use base games data + } + else if (ebxEntry is not null) + { + // add in existing bundles + foreach (int bundle in ebxEntry.Bundles) + { + modifiedBundles.Add(bundle); + } + } + } + } + } + break; + case ResModResource: + break; + case ChunkModResource: + break; + case FsFileModResource: + break; + } + + foreach (int addedBundle in resource.AddedBundles) + { + SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[addedBundle]]; + + if (!sb.Modified.Bundles.TryGetValue(addedBundle, out BundleModInfo? modInfo)) + { + modInfo = new BundleModInfo(); + sb.Modified.Bundles.Add(addedBundle, modInfo); + } + + switch (resource.Type) + { + case ModResourceType.Ebx: + modInfo.Added.Ebx.Add(resource.Name); + break; + case ModResourceType.Res: + modInfo.Added.Res.Add(resource.Name); + break; + case ModResourceType.Chunk: + modInfo.Added.Chunks.Add(Guid.Parse(resource.Name)); + break; + } + } + + foreach (int removedBundle in resource.RemovedBundles) + { + SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[removedBundle]]; + + if (!sb.Modified.Bundles.TryGetValue(removedBundle, out BundleModInfo? modInfo)) + { + modInfo = new BundleModInfo(); + sb.Modified.Bundles.Add(removedBundle, modInfo); + } + + switch (resource.Type) + { + case ModResourceType.Ebx: + modInfo.Removed.Ebx.Add(resource.Name); + break; + case ModResourceType.Res: + modInfo.Removed.Res.Add(resource.Name); + break; + case ModResourceType.Chunk: + modInfo.Removed.Chunks.Add(Guid.Parse(resource.Name)); + break; + } + } + + foreach (int modifiedBundle in modifiedBundles) + { + SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[modifiedBundle]]; + + if (!sb.Modified.Bundles.TryGetValue(modifiedBundle, out BundleModInfo? modInfo)) + { + modInfo = new BundleModInfo(); + sb.Modified.Bundles.Add(modifiedBundle, modInfo); + } + + switch (resource.Type) + { + case ModResourceType.Ebx: + modInfo.Modified.Ebx.Add(resource.Name); + break; + case ModResourceType.Res: + modInfo.Modified.Res.Add(resource.Name); + break; + case ModResourceType.Chunk: + modInfo.Modified.Chunks.Add(Guid.Parse(resource.Name)); + break; + } + } + } + } + + private static List GenerateModInfoList(IEnumerable modPaths) + { + List modInfoList = new(); + + foreach (string path in modPaths) + { + FrostyModDetails modDetails; + + string extension = Path.GetExtension(path); + if (extension == ".fbmod") + { + modDetails = FrostyMod.GetModDetails(path); + } + else if (extension == ".fbcollection") + { + modDetails = FrostyModCollection.GetModDetails(path); + } + else + { + throw new Exception(); + } + + ModInfo modInfo = new() + { + Name = modDetails.Title, + Version = modDetails.Version, + Category = modDetails.Category, + Link = modDetails.ModPageLink, + FileName = path + }; + + modInfoList.Add(modInfo); + } + return modInfoList; + } +} \ No newline at end of file diff --git a/FrostyModSupport/Interfaces/IResourceContainer.cs b/FrostyModSupport/Interfaces/IResourceContainer.cs new file mode 100644 index 000000000..2ef5b7f7f --- /dev/null +++ b/FrostyModSupport/Interfaces/IResourceContainer.cs @@ -0,0 +1,8 @@ +using Frosty.ModSupport.Mod.Resources; + +namespace Frosty.ModSupport.Interfaces; + +public interface IResourceContainer +{ + public IEnumerable Resources { get; } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyMod.cs b/FrostyModSupport/Mod/FrostyMod.cs new file mode 100644 index 000000000..f542afa17 --- /dev/null +++ b/FrostyModSupport/Mod/FrostyMod.cs @@ -0,0 +1,209 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod.Resources; +using Frosty.Sdk; +using Frosty.Sdk.IO; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Utils; + +namespace Frosty.ModSupport.Mod; + +public class FrostyMod : IResourceContainer +{ + /// + /// Mod Format Versions: + /// FBMOD - Initial Version + /// FBMODV1 - (Unknown) + /// FBMODV2 - Special action added for chunks bundles + /// + /// 1 - Start of new binary format + /// - Support for custom data handlers (only for legacy files for now) + /// 2 - Merging of defined res files (eg. ShaderBlockDepot) + /// 3 - Added user data + /// 4 - Various structural changes as well as removal of modifiedBundles + /// 5 - Added link for the ModPage + /// 6 - Storing of Added/Removed Bundles and SuperBundles + /// + public const uint Version = 6; + public const ulong Magic = 0x01005954534F5246; + + public FrostyModDetails ModDetails { get; } + public IEnumerable Resources { get; } + + private FrostyMod(DataStream inStream) + { + + } + + public static FrostyMod Load(string inPath) + { + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + // read header + if (Magic != stream.ReadUInt64()) + { + // not valid need to convert old format or its not a fbmod + } + + if (Version != stream.ReadUInt32()) + { + // we need to convert the mod + } + + long dataOffset = stream.ReadInt64(); + int dataCount = stream.ReadInt32(); + + if (ProfilesLibrary.ProfileName != stream.ReadNullTerminatedString()) + { + // not valid for the loaded profile + } + + if (FileSystemManager.Head != stream.ReadUInt32()) + { + // made for a different version of the game, may or may not work + } + + FrostyModDetails modDetails = new(stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString()); + + // read resources + int resourceCount = stream.ReadInt32(); + BaseModResource[] resources = new BaseModResource[resourceCount]; + for (int i = 0; i < resourceCount; i++) + { + ModResourceType type = (ModResourceType)stream.ReadByte(); + switch (type) + { + case ModResourceType.Embedded: + resources[i] = new EmbeddedModResource(stream); + break; + case ModResourceType.Bundle: + resources[i] = new BundleModResource(stream); + break; + case ModResourceType.Ebx: + resources[i] = new EbxModResource(stream); + break; + case ModResourceType.Res: + resources[i] = new ResModResource(stream); + break; + case ModResourceType.Chunk: + resources[i] = new ChunkModResource(stream); + break; + case ModResourceType.FsFile: + resources[i] = new FsFileModResource(stream); + break; + case ModResourceType.Invalid: + // idk + break; + default: + throw new Exception("Unknown mod resource type"); + } + } + + } + + return default; + } + + public static void Save(string inPath, BaseModResource[] inResources, Block[] inData, + FrostyModDetails inModDetails) + { + int headerSize = sizeof(ulong) + sizeof(uint) + + sizeof(long) + sizeof(int) + + ProfilesLibrary.ProfileName.Length + 1 + sizeof(uint) + + inModDetails.Title.Length + 1 + inModDetails.Author.Length + 1 + + inModDetails.Version.Length + 1 + inModDetails.Description.Length + 1 + + inModDetails.Category.Length + 1 + inModDetails.ModPageLink.Length + 1; + + // TODO: dynamic increasing of block size + Block resources = new(0); + using (DataStream stream = new(new MemoryStream())) + { + stream.WriteInt32(inResources.Length); + + foreach (BaseModResource resource in inResources) + { + resource.Write(stream); + } + } + + // TODO: dynamic increasing of block size + Block datas = new(0); + Block dataHeader = new(inData.Length * (sizeof(long) + sizeof(long))); + using (BlockStream stream = new(dataHeader)) + using (DataStream dataStream = new(new MemoryStream())) + { + foreach (Block data in inData) + { + stream.WriteInt64(dataStream.Position); + stream.WriteInt64(data.Size); + + dataStream.Write(data); + } + } + + Block file = new(headerSize + resources.Size + dataHeader.Size + datas.Size); + using (BlockStream stream = new(file)) + { + stream.WriteUInt64(Magic); + stream.WriteUInt32(Version); + + stream.WriteInt64(headerSize + resources.Size); + stream.WriteInt32(inData.Length); + + stream.WriteNullTerminatedString(ProfilesLibrary.ProfileName); + stream.WriteUInt32(FileSystemManager.Head); + + stream.WriteNullTerminatedString(inModDetails.Title); + stream.WriteNullTerminatedString(inModDetails.Author); + stream.WriteNullTerminatedString(inModDetails.Category); + stream.WriteNullTerminatedString(inModDetails.Version); + stream.WriteNullTerminatedString(inModDetails.Description); + stream.WriteNullTerminatedString(inModDetails.ModPageLink); + + stream.Write(resources); + + stream.Write(dataHeader); + stream.Write(datas); + } + + using (FileStream stream = new(inPath, FileMode.Create, FileAccess.Write)) + { + stream.Write(file); + } + } + + public static FrostyModDetails GetModDetails(string inPath) + { + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + // read header + if (Magic != stream.ReadUInt64()) + { + // not valid need to convert old format or its not a fbmod + } + + if (Version != stream.ReadUInt32()) + { + // we need to convert the mod + } + + long dataOffset = stream.ReadInt64(); + int dataCount = stream.ReadInt32(); + + if (ProfilesLibrary.ProfileName != stream.ReadNullTerminatedString()) + { + // not valid for the loaded profile + } + + if (FileSystemManager.Head != stream.ReadUInt32()) + { + // made for a different version of the game, may or may not work + } + + return new FrostyModDetails(stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString()); + } + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyModCollection.cs b/FrostyModSupport/Mod/FrostyModCollection.cs new file mode 100644 index 000000000..9eea64e58 --- /dev/null +++ b/FrostyModSupport/Mod/FrostyModCollection.cs @@ -0,0 +1,19 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod.Resources; + +namespace Frosty.ModSupport.Mod; + +public class FrostyModCollection : IResourceContainer +{ + public IEnumerable Resources { get; } + + public static FrostyModCollection Load(string inPath) + { + throw new NotImplementedException(); + } + + public static FrostyModDetails GetModDetails(string inPath) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyModDetails.cs b/FrostyModSupport/Mod/FrostyModDetails.cs new file mode 100644 index 000000000..49ba43015 --- /dev/null +++ b/FrostyModSupport/Mod/FrostyModDetails.cs @@ -0,0 +1,51 @@ +namespace Frosty.ModSupport.Mod; + +public sealed class FrostyModDetails +{ + public string Title { get; } + public string Author { get; } + public string Version { get; } + public string Description { get; } + public string Category => string.IsNullOrEmpty(m_category) ? "Misc" : m_category; + public string ModPageLink { get; } + public byte[]? Icon { get; private set; } + + public readonly List Screenshots = new(); + + private readonly string m_category; + + public FrostyModDetails(string inTitle, string inAuthor, string inCategory, string inVersion, string inDescription, string inModPageLink) + { + Title = inTitle; + Author = inAuthor; + Version = inVersion; + Description = inDescription; + m_category = inCategory; + ModPageLink = inModPageLink; + } + + public void SetIcon(Span buffer) + { + Icon = buffer.ToArray(); + } + + public void AddScreenshot(Span buffer) + { + Screenshots.Add(buffer.ToArray()); + } + + public override bool Equals(object? obj) + { + return obj is FrostyModDetails b && Equals(b); + } + + public bool Equals(FrostyModDetails b) + { + return Title == b.Title && Author == b.Author && Version == b.Version && Description == b.Description && Category == b.Category && ModPageLink == b.ModPageLink; + } + + public override int GetHashCode() + { + return HashCode.Combine(Title, Author, Version, Description, Category, ModPageLink); + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/BaseModResource.cs b/FrostyModSupport/Mod/Resources/BaseModResource.cs new file mode 100755 index 000000000..e5040382a --- /dev/null +++ b/FrostyModSupport/Mod/Resources/BaseModResource.cs @@ -0,0 +1,147 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public abstract class BaseModResource +{ + [Flags] + public enum ResourceFlags : byte + { + IsAdded = 1 << 3 + } + + /// + /// The of this . + /// + public virtual ModResourceType Type => ModResourceType.Invalid; + + /// + /// The index into the data array of the or -1 if it doesn't have any data. + /// + public int ResourceIndex { get; } + + /// + /// The name of this . + /// + public string Name { get; } + + /// + /// The hash of the data of this . + /// + public Sha1 Sha1 { get; } + + /// + /// The uncompressed size of the data of this . + /// + public long OriginalSize { get; } + + /// + /// The hash of the handler if this has one, else its 0. + /// + public int HandlerHash { get; } + + /// + /// The user data of this . + /// + public string UserData { get; } = string.Empty; + + /// + /// Indicates if this has data. + /// + public bool IsModified => ResourceIndex != -1 && Type != ModResourceType.Embedded && Type != ModResourceType.Bundle; + + /// + /// The of this . + /// + public ResourceFlags Flags { get; } + + /// + /// Indicates if this has a handler. + /// + public bool HasHandler => HandlerHash != 0; + + /// + /// The bundles this is added to. + /// + public IEnumerable AddedBundles => m_bundlesToAdd; + + /// + /// The bundles this is removed from. + /// + public IEnumerable RemovedBundles => m_bundlesToRemove; + + private readonly HashSet m_bundlesToAdd = new(); + private readonly HashSet m_bundlesToRemove = new(); + + protected BaseModResource(DataStream inStream) + { + ResourceIndex = inStream.ReadInt32(); + Name = inStream.ReadNullTerminatedString(); + + if (ResourceIndex != -1) + { + Sha1 = inStream.ReadSha1(); + OriginalSize = inStream.ReadInt64(); + Flags = (ResourceFlags)inStream.ReadByte(); + HandlerHash = inStream.ReadInt32(); + UserData = inStream.ReadNullTerminatedString(); + } + + int addCount = inStream.ReadInt32(); + for (int i = 0; i < addCount; i++) + { + m_bundlesToAdd.Add(inStream.ReadInt32()); + } + + int removeCount = inStream.ReadInt32(); + for (int i = 0; i < removeCount; i++) + { + m_bundlesToRemove.Add(inStream.ReadInt32()); + } + } + + protected BaseModResource(int inResourceIndex, string inName, Sha1 inSha1, long inOriginalSize, + ResourceFlags inFlags, int inHandlerHash, string inUserData, IEnumerable inBundlesToAdd, + IEnumerable inBundlesToRemove) + { + ResourceIndex = inResourceIndex; + Name = inName; + Sha1 = inSha1; + OriginalSize = inOriginalSize; + Flags = inFlags; + HandlerHash = inHandlerHash; + UserData = inUserData; + m_bundlesToAdd.UnionWith(inBundlesToAdd); + m_bundlesToRemove.UnionWith(inBundlesToRemove); + } + + public virtual void Write(DataStream stream) + { + stream.WriteByte((byte)Type); + stream.WriteInt32(ResourceIndex); + stream.WriteNullTerminatedString(Name); + + if (ResourceIndex != -1) + { + stream.WriteSha1(Sha1); + stream.WriteInt64(OriginalSize); + stream.WriteByte((byte)Flags); + stream.WriteInt32(HandlerHash); + stream.WriteNullTerminatedString(UserData); + } + + stream.WriteInt32(m_bundlesToAdd.Count); + foreach (int bundleId in m_bundlesToAdd) + { + stream.WriteInt32(bundleId); + } + + stream.WriteInt32(m_bundlesToRemove.Count); + foreach (int bundleId in m_bundlesToRemove) + { + stream.WriteInt32(bundleId); + } + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/BundleModResource.cs b/FrostyModSupport/Mod/Resources/BundleModResource.cs new file mode 100755 index 000000000..ba9a02878 --- /dev/null +++ b/FrostyModSupport/Mod/Resources/BundleModResource.cs @@ -0,0 +1,34 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public class BundleModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.Bundle; + + /// + /// Fnv hash of the lowercase SuperBundle name of this bundle. + /// + public int SuperBundleHash { get; } + + public BundleModResource(DataStream inStream) + : base(inStream) + { + SuperBundleHash = inStream.ReadInt32(); + } + + internal BundleModResource(string inName, int inSuperBundleHash) + : base(-1, inName, Sha1.Zero, 0, 0, 0, + string.Empty, Enumerable.Empty(), Enumerable.Empty()) + { + SuperBundleHash = inSuperBundleHash; + } + + public override void Write(DataStream stream) + { + base.Write(stream); + + stream.WriteInt32(SuperBundleHash); + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/ChunkModResource.cs b/FrostyModSupport/Mod/Resources/ChunkModResource.cs new file mode 100755 index 000000000..e3c24177a --- /dev/null +++ b/FrostyModSupport/Mod/Resources/ChunkModResource.cs @@ -0,0 +1,86 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public class ChunkModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.Chunk; + + public uint RangeStart { get; } + public uint RangeEnd { get; } + public uint LogicalOffset { get; } + public uint LogicalSize { get; } + public int H32 { get; } + public int FirstMip { get; } + public IEnumerable AddedSuperBundles => m_superBundlesToAdd; + public IEnumerable RemovedSuperBundles => m_superBundlesToRemove; + + protected HashSet m_superBundlesToAdd = new(); + protected HashSet m_superBundlesToRemove = new(); + + public ChunkModResource(DataStream inStream) + : base(inStream) + { + RangeStart = inStream.ReadUInt32(); + RangeEnd = inStream.ReadUInt32(); + LogicalOffset = inStream.ReadUInt32(); + LogicalSize = inStream.ReadUInt32(); + H32 = inStream.ReadInt32(); + FirstMip = inStream.ReadInt32(); + + int addedCount = inStream.ReadInt32(); + for (int i = 0; i < addedCount; i++) + { + m_superBundlesToAdd.Add(inStream.ReadInt32()); + } + + int removedCount = inStream.ReadInt32(); + for (int i = 0; i < removedCount; i++) + { + m_superBundlesToRemove.Add(inStream.ReadInt32()); + } + } + + public ChunkModResource(int inResourceIndex, string inName, Sha1 inSha1, long inOriginalSize, + ResourceFlags inFlags, int inHandlerHash, string inUserData, IEnumerable inBundlesToAdd, + IEnumerable inBundlesToRemove, uint inRangeStart, uint inRangeEnd, uint inLogicalOffset, + uint inLogicalSize, int inH32, int inFirstMip, IEnumerable inSuperBundlesToAdd, + IEnumerable inSuperBundlesToRemove) + : base(inResourceIndex, inName, inSha1, inOriginalSize, inFlags, inHandlerHash, + inUserData, inBundlesToAdd, inBundlesToRemove) + { + RangeStart = inRangeStart; + RangeEnd = inRangeEnd; + LogicalOffset = inLogicalOffset; + LogicalSize = inLogicalSize; + H32 = inH32; + FirstMip = inFirstMip; + m_superBundlesToAdd.UnionWith(inSuperBundlesToAdd); + m_superBundlesToRemove.UnionWith(inSuperBundlesToRemove); + } + + public override void Write(DataStream stream) + { + base.Write(stream); + + stream.WriteUInt32(RangeStart); + stream.WriteUInt32(RangeEnd); + stream.WriteUInt32(LogicalOffset); + stream.WriteUInt32(LogicalSize); + stream.WriteInt32(H32); + stream.WriteInt32(FirstMip); + + stream.WriteInt32(m_superBundlesToAdd.Count); + foreach (int superBundleId in m_superBundlesToAdd) + { + stream.WriteInt32(superBundleId); + } + + stream.WriteInt32(m_superBundlesToRemove.Count); + foreach (int superBundleId in m_superBundlesToRemove) + { + stream.WriteInt32(superBundleId); + } + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/EbxModResource.cs b/FrostyModSupport/Mod/Resources/EbxModResource.cs new file mode 100755 index 000000000..dbb801c79 --- /dev/null +++ b/FrostyModSupport/Mod/Resources/EbxModResource.cs @@ -0,0 +1,22 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public class EbxModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.Ebx; + + public EbxModResource(DataStream inStream) + : base(inStream) + { + } + + public EbxModResource(int inResourceIndex, string inName, Sha1 inSha1, long inOriginalSize, + ResourceFlags inFlags, int inHandlerHash, string inUserData, IEnumerable inBundlesToAdd, + IEnumerable inBundlesToRemove) + : base(inResourceIndex, inName, inSha1, inOriginalSize, inFlags, inHandlerHash, + inUserData, inBundlesToAdd, inBundlesToRemove) + { + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs b/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs new file mode 100755 index 000000000..137d47cb1 --- /dev/null +++ b/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs @@ -0,0 +1,20 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public sealed class EmbeddedModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.Embedded; + + public EmbeddedModResource(DataStream inStream) + : base(inStream) + { + } + + internal EmbeddedModResource(int inResourceIndex, string inName) + : base(inResourceIndex, inName, Sha1.Zero, 0, 0, 0, + string.Empty, Enumerable.Empty(), Enumerable.Empty()) + { + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/FsFileModResource.cs b/FrostyModSupport/Mod/Resources/FsFileModResource.cs new file mode 100755 index 000000000..5a512fb37 --- /dev/null +++ b/FrostyModSupport/Mod/Resources/FsFileModResource.cs @@ -0,0 +1,13 @@ +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public class FsFileModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.FsFile; + + public FsFileModResource(DataStream inStream) + : base(inStream) + { + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/ModResourceType.cs b/FrostyModSupport/Mod/Resources/ModResourceType.cs new file mode 100755 index 000000000..f2fadae72 --- /dev/null +++ b/FrostyModSupport/Mod/Resources/ModResourceType.cs @@ -0,0 +1,42 @@ +namespace Frosty.ModSupport.Mod.Resources; + +/// +/// Represents the type of the data the resource represents. +/// +public enum ModResourceType +{ + /// + /// Invalid resource type. + /// + Invalid = -1, + + /// + /// Embedded data such as icons or images. + /// + Embedded, + + /// + /// Data relating to ebx assets. + /// + Ebx, + + /// + /// Data relating to resources. + /// + Res, + + /// + /// Data relating to chunks. + /// + Chunk, + + /// + /// Data relating to bundles. + /// + Bundle, + + /// + /// Data relating to initfs files. + /// + FsFile +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/ResModResource.cs b/FrostyModSupport/Mod/Resources/ResModResource.cs new file mode 100755 index 000000000..e1baf8b8c --- /dev/null +++ b/FrostyModSupport/Mod/Resources/ResModResource.cs @@ -0,0 +1,42 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; + +namespace Frosty.ModSupport.Mod.Resources; + +public class ResModResource : BaseModResource +{ + public override ModResourceType Type => ModResourceType.Res; + + public uint ResType { get; } + public ulong ResRid { get; } + public byte[] ResMeta { get; } + + public ResModResource(DataStream inStream) + : base(inStream) + { + ResType = inStream.ReadUInt32(); + ResRid = inStream.ReadUInt64(); + ResMeta = inStream.ReadBytes(inStream.ReadInt32()); + } + + public ResModResource(int inResourceIndex, string inName, Sha1 inSha1, long inOriginalSize, + ResourceFlags inFlags, int inHandlerHash, string inUserData, IEnumerable inBundlesToAdd, + IEnumerable inBundlesToRemove, uint inResType, ulong inResRid, byte[] inResMeta) + : base(inResourceIndex, inName, inSha1, inOriginalSize, inFlags, inHandlerHash, + inUserData, inBundlesToAdd, inBundlesToRemove) + { + ResType = inResType; + ResRid = inResRid; + ResMeta = inResMeta; + } + + public override void Write(DataStream stream) + { + base.Write(stream); + + stream.WriteUInt32(ResType); + stream.WriteUInt64(ResRid); + stream.WriteInt32(ResMeta.Length); + stream.Write(ResMeta); + } +} \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ChunkModEntry.cs b/FrostyModSupport/ModEntries/ChunkModEntry.cs new file mode 100644 index 000000000..3a837d139 --- /dev/null +++ b/FrostyModSupport/ModEntries/ChunkModEntry.cs @@ -0,0 +1,6 @@ +namespace Frosty.ModSupport.ModEntries; + +public class ChunkModEntry +{ + +} \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/EbxModEntry.cs b/FrostyModSupport/ModEntries/EbxModEntry.cs new file mode 100644 index 000000000..b94cebc5a --- /dev/null +++ b/FrostyModSupport/ModEntries/EbxModEntry.cs @@ -0,0 +1,12 @@ +using Frosty.Sdk; + +namespace Frosty.ModSupport.ModEntries; + +public class EbxModEntry +{ + public string Name { get; } + public Sha1 Sha1 { get; } + public long OriginalSize { get; } + public long Size { get; } + +} \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ResModEntry.cs b/FrostyModSupport/ModEntries/ResModEntry.cs new file mode 100644 index 000000000..7356d3996 --- /dev/null +++ b/FrostyModSupport/ModEntries/ResModEntry.cs @@ -0,0 +1,6 @@ +namespace Frosty.ModSupport.ModEntries; + +public class ResModEntry +{ + +} \ No newline at end of file diff --git a/FrostyModSupport/ModInfo.cs b/FrostyModSupport/ModInfo.cs new file mode 100644 index 000000000..cc03f80b6 --- /dev/null +++ b/FrostyModSupport/ModInfo.cs @@ -0,0 +1,30 @@ +namespace Frosty.ModSupport; + +public class ModInfo +{ + public string Name { get; set; } = string.Empty; + public string Version { get; set; } = string.Empty; + public string Category { get; set; } = string.Empty; + public string Link { get; set; } = string.Empty; + public string FileName { get; set; } = string.Empty; + + public override bool Equals(object? obj) + { + if (obj is ModInfo other) + { + return Equals(other); + } + + return false; + } + + public bool Equals(ModInfo other) + { + return Name == other.Name && Version == other.Version && Category == other.Category && FileName == other.FileName; + } + + public override int GetHashCode() + { + return HashCode.Combine(Name, Version, Category, Link, FileName); + } +} \ No newline at end of file diff --git a/FrostyModSupport/ModInfos/BundleModAction.cs b/FrostyModSupport/ModInfos/BundleModAction.cs index ca3e378e9..56df1d6b4 100644 --- a/FrostyModSupport/ModInfos/BundleModAction.cs +++ b/FrostyModSupport/ModInfos/BundleModAction.cs @@ -2,7 +2,7 @@ public class BundleModAction { - public List Ebx = new(); - public List Res = new(); - public List Chunks = new(); + public HashSet Ebx = new(); + public HashSet Res = new(); + public HashSet Chunks = new(); } \ No newline at end of file diff --git a/FrostyModSupport/ModInfos/SuperBundleModAction.cs b/FrostyModSupport/ModInfos/SuperBundleModAction.cs index 01d9468fa..44d1f19a3 100644 --- a/FrostyModSupport/ModInfos/SuperBundleModAction.cs +++ b/FrostyModSupport/ModInfos/SuperBundleModAction.cs @@ -2,6 +2,6 @@ public class SuperBundleModAction { - public List Bundles = new(); - public List Chunks = new(); + public Dictionary Bundles = new(); + public HashSet Chunks = new(); } \ No newline at end of file From dd2b4d7b987ac1fe85111f5e3927ca9764159abb Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:37:01 +0200 Subject: [PATCH 02/14] [ModSupport] Added RetCode and fixed mod writing --- FrostyModSupport/FrostyModExecutor.cs | 29 ++++++++++--- FrostyModSupport/Mod/FrostyMod.cs | 48 +++++++++++++-------- FrostyModSupport/Mod/FrostyModCollection.cs | 2 +- FrostyModSupport/RetCode.cs | 8 ++++ 4 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 FrostyModSupport/RetCode.cs diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 5f739a5c3..1fe416b6b 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -24,7 +24,7 @@ public class FrostyModExecutor /// /// The name of the directory where the data is stored in the games ModData folder. /// The full paths of the mods. - public void GenerateMods(string modPackName, params string[] modPaths) + public RetCode GenerateMods(string modPackName, params string[] modPaths) { string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName); string patchPath = FileSystemManager.Sources.Count == 1 @@ -39,7 +39,7 @@ public void GenerateMods(string modPackName, params string[] modPaths) List? oldModInfos = JsonSerializer.Deserialize>(File.ReadAllText(modInfosPath)); if (oldModInfos?.SequenceEqual(modInfos) == true) { - return; + return RetCode.NoUpdateNeeded; } } @@ -56,19 +56,31 @@ public void GenerateMods(string modPackName, params string[] modPaths) string extension = Path.GetExtension(path); if (extension == ".fbmod") { - FrostyMod mod = FrostyMod.Load(path); + FrostyMod? mod = FrostyMod.Load(path); + if (mod is null) + { + return RetCode.InvalidMods; + } ProcessModResources(mod); } else if (extension == ".fbcollection") { - FrostyModCollection modCollection = FrostyModCollection.Load(path); + FrostyModCollection? modCollection = FrostyModCollection.Load(path); + if (modCollection is null) + { + return RetCode.InvalidMods; + } ProcessModResources(modCollection); } else { - throw new Exception(); + return RetCode.InvalidMods; } } + + + + return RetCode.Success; } private void GenerateBundleLookup() @@ -223,7 +235,7 @@ private static List GenerateModInfoList(IEnumerable modPaths) foreach (string path in modPaths) { - FrostyModDetails modDetails; + FrostyModDetails? modDetails; string extension = Path.GetExtension(path); if (extension == ".fbmod") @@ -238,6 +250,11 @@ private static List GenerateModInfoList(IEnumerable modPaths) { throw new Exception(); } + + if (modDetails is null) + { + return modInfoList; + } ModInfo modInfo = new() { diff --git a/FrostyModSupport/Mod/FrostyMod.cs b/FrostyModSupport/Mod/FrostyMod.cs index f542afa17..d23afb4a1 100644 --- a/FrostyModSupport/Mod/FrostyMod.cs +++ b/FrostyModSupport/Mod/FrostyMod.cs @@ -34,19 +34,19 @@ private FrostyMod(DataStream inStream) } - public static FrostyMod Load(string inPath) + public static FrostyMod? Load(string inPath) { using (BlockStream stream = BlockStream.FromFile(inPath, false)) { // read header if (Magic != stream.ReadUInt64()) { - // not valid need to convert old format or its not a fbmod + return null; } if (Version != stream.ReadUInt32()) { - // we need to convert the mod + return null; } long dataOffset = stream.ReadInt64(); @@ -54,7 +54,7 @@ public static FrostyMod Load(string inPath) if (ProfilesLibrary.ProfileName != stream.ReadNullTerminatedString()) { - // not valid for the loaded profile + return null; } if (FileSystemManager.Head != stream.ReadUInt32()) @@ -115,8 +115,7 @@ public static void Save(string inPath, BaseModResource[] inResources, Block resources = new(0); + Block resources; using (DataStream stream = new(new MemoryStream())) { stream.WriteInt32(inResources.Length); @@ -125,25 +124,32 @@ public static void Save(string inPath, BaseModResource[] inResources, Block((int)stream.Length); + stream.ReadExactly(resources); } - // TODO: dynamic increasing of block size - Block datas = new(0); + Block data; Block dataHeader = new(inData.Length * (sizeof(long) + sizeof(long))); - using (BlockStream stream = new(dataHeader)) + using (BlockStream stream = new(dataHeader, true)) using (DataStream dataStream = new(new MemoryStream())) { - foreach (Block data in inData) + foreach (Block subData in inData) { stream.WriteInt64(dataStream.Position); - stream.WriteInt64(data.Size); + stream.WriteInt64(subData.Size); - dataStream.Write(data); + dataStream.Write(subData); } + + dataStream.Position = 0; + data = new Block((int)dataStream.Length); + dataStream.ReadExactly(data); } - Block file = new(headerSize + resources.Size + dataHeader.Size + datas.Size); - using (BlockStream stream = new(file)) + Block file = new(headerSize + resources.Size + dataHeader.Size + data.Size); + using (BlockStream stream = new(file, true)) { stream.WriteUInt64(Magic); stream.WriteUInt32(Version); @@ -162,30 +168,34 @@ public static void Save(string inPath, BaseModResource[] inResources, Block Resources { get; } - public static FrostyModCollection Load(string inPath) + public static FrostyModCollection? Load(string inPath) { throw new NotImplementedException(); } diff --git a/FrostyModSupport/RetCode.cs b/FrostyModSupport/RetCode.cs new file mode 100644 index 000000000..04832a2fa --- /dev/null +++ b/FrostyModSupport/RetCode.cs @@ -0,0 +1,8 @@ +namespace Frosty.ModSupport; + +public enum RetCode +{ + NoUpdateNeeded = 1, + Success = 0, + InvalidMods = -1, +} \ No newline at end of file From 1cbc904fe0d227067dc6e17c16c4479461ea4475 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:37:42 +0200 Subject: [PATCH 03/14] [Sdk] Added leave open option to BlockStream --- FrostySdk/IO/BlockStream.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/FrostySdk/IO/BlockStream.cs b/FrostySdk/IO/BlockStream.cs index da23c7c07..2a166a08b 100644 --- a/FrostySdk/IO/BlockStream.cs +++ b/FrostySdk/IO/BlockStream.cs @@ -11,6 +11,7 @@ namespace Frosty.Sdk.IO; public class BlockStream : DataStream { private readonly Block m_block; + private bool m_leaveOpen; public BlockStream(int inSize) { @@ -18,10 +19,11 @@ public BlockStream(int inSize) m_stream = m_block.ToStream(); } - public BlockStream(Block inBuffer) + public BlockStream(Block inBuffer, bool inLeaveOpen = false) { m_block = inBuffer; m_stream = m_block.ToStream(); + m_leaveOpen = inLeaveOpen; } public override unsafe string ReadNullTerminatedString(bool wide = false) @@ -181,7 +183,10 @@ public void Decrypt(byte[] inKey, int inSize, PaddingMode inPaddingMode) public override void Dispose() { base.Dispose(); - m_block.Dispose(); + if (!m_leaveOpen) + { + m_block.Dispose(); + } GC.SuppressFinalize(this); } From 10407bccf66c46a057979b6540fb9c45865c84e6 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:57:24 +0200 Subject: [PATCH 04/14] Small changes with IResourceContainer.cs --- FrostyModSupport/{RetCode.cs => Errors.cs} | 2 +- FrostyModSupport/FrostyModExecutor.cs | 71 ++++-- .../Interfaces/IResourceContainer.cs | 11 + FrostyModSupport/Mod/FrostyMod.cs | 202 ++++++++++++++--- FrostyModSupport/Mod/FrostyModCollection.cs | 7 +- FrostyModSupport/Mod/ModUpdater.cs | 203 ++++++++++++++++++ .../Mod/Resources/EmbeddedModResource.cs | 2 +- 7 files changed, 446 insertions(+), 52 deletions(-) rename FrostyModSupport/{RetCode.cs => Errors.cs} (82%) create mode 100644 FrostyModSupport/Mod/ModUpdater.cs diff --git a/FrostyModSupport/RetCode.cs b/FrostyModSupport/Errors.cs similarity index 82% rename from FrostyModSupport/RetCode.cs rename to FrostyModSupport/Errors.cs index 04832a2fa..6d7bc2877 100644 --- a/FrostyModSupport/RetCode.cs +++ b/FrostyModSupport/Errors.cs @@ -1,6 +1,6 @@ namespace Frosty.ModSupport; -public enum RetCode +public enum Errors { NoUpdateNeeded = 1, Success = 0, diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 1fe416b6b..6d02dc7b7 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -24,7 +24,7 @@ public class FrostyModExecutor /// /// The name of the directory where the data is stored in the games ModData folder. /// The full paths of the mods. - public RetCode GenerateMods(string modPackName, params string[] modPaths) + public Errors GenerateMods(string modPackName, params string[] modPaths) { string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName); string patchPath = FileSystemManager.Sources.Count == 1 @@ -39,7 +39,7 @@ public RetCode GenerateMods(string modPackName, params string[] modPaths) List? oldModInfos = JsonSerializer.Deserialize>(File.ReadAllText(modInfosPath)); if (oldModInfos?.SequenceEqual(modInfos) == true) { - return RetCode.NoUpdateNeeded; + return Errors.NoUpdateNeeded; } } @@ -59,7 +59,11 @@ public RetCode GenerateMods(string modPackName, params string[] modPaths) FrostyMod? mod = FrostyMod.Load(path); if (mod is null) { - return RetCode.InvalidMods; + return Errors.InvalidMods; + } + if (mod.Head != FileSystemManager.Head) + { + // TODO: print warning } ProcessModResources(mod); } @@ -68,19 +72,40 @@ public RetCode GenerateMods(string modPackName, params string[] modPaths) FrostyModCollection? modCollection = FrostyModCollection.Load(path); if (modCollection is null) { - return RetCode.InvalidMods; + return Errors.InvalidMods; + } + + foreach (FrostyMod mod in modCollection.Mods) + { + if (mod.Head != FileSystemManager.Head) + { + // TODO: print warning + } + ProcessModResources(mod); } - ProcessModResources(modCollection); } else { - return RetCode.InvalidMods; + return Errors.InvalidMods; } } - - - return RetCode.Success; + foreach (KeyValuePair sb in m_superBundleModInfos) + { + switch (FileSystemManager.BundleFormat) + { + case BundleFormat.Dynamic2018: + break; + case BundleFormat.Manifest2019: + break; + case BundleFormat.SuperBundleManifest: + break; + case BundleFormat.Kelvin: + break; + } + } + + return Errors.Success; } private void GenerateBundleLookup() @@ -149,7 +174,10 @@ private void ProcessModResources(IResourceContainer container) break; case ResModResource: break; - case ChunkModResource: + case ChunkModResource chunk: + foreach (int superBundle in chunk.AddedSuperBundles) + { + } break; case FsFileModResource: break; @@ -157,7 +185,12 @@ private void ProcessModResources(IResourceContainer container) foreach (int addedBundle in resource.AddedBundles) { - SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[addedBundle]]; + string sbName = m_mapping[addedBundle]; + if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) + { + sb = new SuperBundleModInfo(); + m_superBundleModInfos.Add(sbName, sb); + } if (!sb.Modified.Bundles.TryGetValue(addedBundle, out BundleModInfo? modInfo)) { @@ -181,7 +214,12 @@ private void ProcessModResources(IResourceContainer container) foreach (int removedBundle in resource.RemovedBundles) { - SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[removedBundle]]; + string sbName = m_mapping[removedBundle]; + if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) + { + sb = new SuperBundleModInfo(); + m_superBundleModInfos.Add(sbName, sb); + } if (!sb.Modified.Bundles.TryGetValue(removedBundle, out BundleModInfo? modInfo)) { @@ -205,8 +243,13 @@ private void ProcessModResources(IResourceContainer container) foreach (int modifiedBundle in modifiedBundles) { - SuperBundleModInfo sb = m_superBundleModInfos[m_mapping[modifiedBundle]]; - + string sbName = m_mapping[modifiedBundle]; + if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) + { + sb = new SuperBundleModInfo(); + m_superBundleModInfos.Add(sbName, sb); + } + if (!sb.Modified.Bundles.TryGetValue(modifiedBundle, out BundleModInfo? modInfo)) { modInfo = new BundleModInfo(); diff --git a/FrostyModSupport/Interfaces/IResourceContainer.cs b/FrostyModSupport/Interfaces/IResourceContainer.cs index 2ef5b7f7f..c273970ba 100644 --- a/FrostyModSupport/Interfaces/IResourceContainer.cs +++ b/FrostyModSupport/Interfaces/IResourceContainer.cs @@ -1,8 +1,19 @@ using Frosty.ModSupport.Mod.Resources; +using Frosty.Sdk.Utils; namespace Frosty.ModSupport.Interfaces; public interface IResourceContainer { + /// + /// The Resources of this resource container. + /// public IEnumerable Resources { get; } + + /// + /// Gets the data of a resource + /// + /// The index of the resource. + /// + public Block GetData(int inIndex); } \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyMod.cs b/FrostyModSupport/Mod/FrostyMod.cs index d23afb4a1..9b9fc0b99 100644 --- a/FrostyModSupport/Mod/FrostyMod.cs +++ b/FrostyModSupport/Mod/FrostyMod.cs @@ -9,6 +9,32 @@ namespace Frosty.ModSupport.Mod; public class FrostyMod : IResourceContainer { + public class ResourceData + { + private string m_fileName; + private long m_offset; + private int m_size; + + public ResourceData(string inFileName, long inOffset, int inSize) + { + m_fileName = inFileName; + m_offset = inOffset; + m_size = inSize; + } + + public Block GetData() + { + Block retVal = new(m_size); + using (FileStream stream = new(m_fileName, FileMode.Open, FileAccess.Read)) + { + stream.Position = m_offset; + stream.ReadExactly(retVal); + } + + return retVal; + } + } + /// /// Mod Format Versions: /// FBMOD - Initial Version @@ -26,14 +52,34 @@ public class FrostyMod : IResourceContainer public const uint Version = 6; public const ulong Magic = 0x01005954534F5246; + + public IEnumerable Resources => m_resources; + public FrostyModDetails ModDetails { get; } - public IEnumerable Resources { get; } + + public uint Head { get; } - private FrostyMod(DataStream inStream) + private BaseModResource[] m_resources; + private ResourceData[] m_data; + + private FrostyMod(FrostyModDetails inModDetails, uint inHead, BaseModResource[] inResources, ResourceData[] inData) { - + ModDetails = inModDetails; + m_resources = inResources; + m_data = inData; + } + + public Block GetData(int inIndex) + { + return m_data[inIndex].GetData(); } + /// + /// Loads a mod from a file. + /// + /// The path to the file. + /// The mod or null if its not a fbmod/an older format fbmod/ fbmod made for another profile. + /// When the mod file is corrupted. public static FrostyMod? Load(string inPath) { using (BlockStream stream = BlockStream.FromFile(inPath, false)) @@ -57,10 +103,7 @@ private FrostyMod(DataStream inStream) return null; } - if (FileSystemManager.Head != stream.ReadUInt32()) - { - // made for a different version of the game, may or may not work - } + uint head = stream.ReadUInt32(); FrostyModDetails modDetails = new(stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), @@ -99,12 +142,65 @@ private FrostyMod(DataStream inStream) throw new Exception("Unknown mod resource type"); } } - - } + + // read data + stream.Position = dataOffset; + ResourceData[] data = new ResourceData[dataCount]; + for (int i = 0; i < dataCount; i++) + { + data[i] = new ResourceData(inPath, stream.ReadInt64(), stream.ReadInt32()); + } - return default; + return new FrostyMod(modDetails, head, resources, data); + } + } + + /// + /// Gets the ModDetails of a mod. + /// + /// The file path of the mod. + /// The of that mod. + public static FrostyModDetails? GetModDetails(string inPath) + { + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + // read header + if (Magic != stream.ReadUInt64()) + { + return null; + } + + if (Version != stream.ReadUInt32()) + { + return null; + } + + long dataOffset = stream.ReadInt64(); + int dataCount = stream.ReadInt32(); + + if (ProfilesLibrary.ProfileName != stream.ReadNullTerminatedString()) + { + return null; + } + + if (FileSystemManager.Head != stream.ReadUInt32()) + { + // made for a different version of the game, may or may not work + } + + return new FrostyModDetails(stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), + stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString()); + } } + /// + /// Saves a mod to a file. + /// + /// The path of the file. + /// The resources of the mod. + /// The data of the resources. + /// The details of the mod. public static void Save(string inPath, BaseModResource[] inResources, Block[] inData, FrostyModDetails inModDetails) { @@ -183,37 +279,81 @@ public static void Save(string inPath, BaseModResource[] inResources, Block[] inData, + FrostyModDetails inModDetails, uint inHead) { - using (BlockStream stream = BlockStream.FromFile(inPath, false)) + int headerSize = sizeof(ulong) + sizeof(uint) + + sizeof(long) + sizeof(int) + + ProfilesLibrary.ProfileName.Length + 1 + sizeof(uint) + + inModDetails.Title.Length + 1 + inModDetails.Author.Length + 1 + + inModDetails.Version.Length + 1 + inModDetails.Description.Length + 1 + + inModDetails.Category.Length + 1 + inModDetails.ModPageLink.Length + 1; + + Block resources; + using (DataStream stream = new(new MemoryStream())) { - // read header - if (Magic != stream.ReadUInt64()) - { - return null; - } + stream.WriteInt32(inResources.Length); - if (Version != stream.ReadUInt32()) + foreach (BaseModResource resource in inResources) { - return null; + resource.Write(stream); } - long dataOffset = stream.ReadInt64(); - int dataCount = stream.ReadInt32(); - - if (ProfilesLibrary.ProfileName != stream.ReadNullTerminatedString()) + stream.Position = 0; + resources = new Block((int)stream.Length); + stream.ReadExactly(resources); + } + + Block data; + Block dataHeader = new(inData.Length * (sizeof(long) + sizeof(long))); + using (BlockStream stream = new(dataHeader, true)) + using (DataStream dataStream = new(new MemoryStream())) + { + foreach (Block subData in inData) { - return null; + stream.WriteInt64(dataStream.Position); + stream.WriteInt32(subData.Size); + + dataStream.Write(subData); } - if (FileSystemManager.Head != stream.ReadUInt32()) - { - // made for a different version of the game, may or may not work - } + dataStream.Position = 0; + data = new Block((int)dataStream.Length); + dataStream.ReadExactly(data); + } + + Block file = new(headerSize + resources.Size + dataHeader.Size + data.Size); + using (BlockStream stream = new(file, true)) + { + stream.WriteUInt64(Magic); + stream.WriteUInt32(Version); + + stream.WriteInt64(headerSize + resources.Size); + stream.WriteInt32(inData.Length); + + stream.WriteNullTerminatedString(ProfilesLibrary.ProfileName); + stream.WriteUInt32(inHead); + + stream.WriteNullTerminatedString(inModDetails.Title); + stream.WriteNullTerminatedString(inModDetails.Author); + stream.WriteNullTerminatedString(inModDetails.Category); + stream.WriteNullTerminatedString(inModDetails.Version); + stream.WriteNullTerminatedString(inModDetails.Description); + stream.WriteNullTerminatedString(inModDetails.ModPageLink); + + stream.Write(resources); + resources.Dispose(); + + stream.Write(dataHeader); + stream.Write(data); + dataHeader.Dispose(); + data.Dispose(); + } - return new FrostyModDetails(stream.ReadNullTerminatedString(), - stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), - stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString()); + using (FileStream stream = new(inPath, FileMode.Create, FileAccess.Write)) + { + stream.Write(file); + file.Dispose(); } } } \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyModCollection.cs b/FrostyModSupport/Mod/FrostyModCollection.cs index 911f6e169..47d55fd88 100644 --- a/FrostyModSupport/Mod/FrostyModCollection.cs +++ b/FrostyModSupport/Mod/FrostyModCollection.cs @@ -1,11 +1,8 @@ -using Frosty.ModSupport.Interfaces; -using Frosty.ModSupport.Mod.Resources; - namespace Frosty.ModSupport.Mod; -public class FrostyModCollection : IResourceContainer +public class FrostyModCollection { - public IEnumerable Resources { get; } + public IEnumerable Mods { get; } public static FrostyModCollection? Load(string inPath) { diff --git a/FrostyModSupport/Mod/ModUpdater.cs b/FrostyModSupport/Mod/ModUpdater.cs new file mode 100644 index 000000000..408d166e6 --- /dev/null +++ b/FrostyModSupport/Mod/ModUpdater.cs @@ -0,0 +1,203 @@ +using Frosty.ModSupport.Mod.Resources; +using Frosty.Sdk; +using Frosty.Sdk.DbObjectElements; +using Frosty.Sdk.IO; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Utils; + +namespace Frosty.ModSupport.Mod; + +public class ModUpdater +{ + private static Dictionary> s_bundleMapping = new(); + + public static Errors UpdateMod(string inPath) + { + (BaseModResource[], Block[], FrostyModDetails, uint)? mod; + + string extension = Path.GetExtension(inPath); + + if (extension == ".daimod") + { + // TODO: daimod convert + } + else if (extension != ".fbmod") + { + return Errors.InvalidMods; + } + + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + // read header + if (FrostyMod.Magic != stream.ReadUInt64()) + { + stream.Position = 0; + mod = UpdateLegacyFormat(stream, inPath.Replace(".fbmod", string.Empty)); + } + else + { + mod = UpdateNewFormat(stream); + } + } + + if (mod is null) + { + return Errors.InvalidMods; + } + + FrostyMod.Save(inPath, mod.Value.Item1, mod.Value.Item2, mod.Value.Item3, mod.Value.Item4); + + foreach (Block block in mod.Value.Item2) + { + block.Dispose(); + } + + return Errors.Success; + } + + private static (BaseModResource[], Block[], FrostyModDetails, uint)? UpdateNewFormat(DataStream inStream) + { + uint version = inStream.ReadUInt32(); + + long dataOffset = inStream.ReadInt64(); + int dataCount = inStream.ReadInt32(); + + if (ProfilesLibrary.ProfileName.Equals(inStream.ReadSizedString(), StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + (BaseModResource[], Block[], FrostyModDetails, uint) retVal = default; + retVal.Item4 = inStream.ReadUInt32(); + + retVal.Item3 = new FrostyModDetails(inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), + inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), + version > 4 ? inStream.ReadNullTerminatedString() : string.Empty); + + int resourceCount = inStream.ReadInt32(); + BaseModResource[] resources = new BaseModResource[resourceCount]; + for (int i = 0; i < resourceCount; i++) + { + ModResourceType type = (ModResourceType)inStream.ReadByte(); + (int, string, Sha1, long, int, string, IEnumerable) b; + BaseModResource.ResourceFlags flags; + switch (type) + { + case ModResourceType.Embedded: + b = ReadBaseModResource(inStream, version); + resources[i] = new EmbeddedModResource(b.Item1, b.Item2); + break; + case ModResourceType.Bundle: + b = ReadBaseModResource(inStream, version); + inStream.ReadNullTerminatedString(); + int superBundleHash = inStream.ReadInt32(); + // TODO: update hash and add bundle hash to s_bundleMapping + + resources[i] = new BundleModResource(b.Item2, superBundleHash); + break; + case ModResourceType.Ebx: + b = ReadBaseModResource(inStream, version); + flags = AssetManager.GetEbxAssetEntry(b.Item2) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; + + resources[i] = new EbxModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, b.Item7, Enumerable.Empty()); + break; + case ModResourceType.Res: + b = ReadBaseModResource(inStream, version); + flags = AssetManager.GetResAssetEntry(b.Item2) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; + + resources[i] = new ResModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, b.Item7, + Enumerable.Empty(), inStream.ReadUInt32(), inStream.ReadUInt64(), + inStream.ReadBytes(inStream.ReadInt32())); + break; + case ModResourceType.Chunk: + b = ReadBaseModResource(inStream, version); + flags = AssetManager.GetChunkAssetEntry(Guid.Parse(b.Item2)) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; + + uint rangeStart = inStream.ReadUInt32(); + uint rangeEnd = inStream.ReadUInt32(); + uint logicalOffset = inStream.ReadUInt32(); + uint logicalSize = inStream.ReadUInt32(); + int h32 = inStream.ReadInt32(); + int firstMip = inStream.ReadInt32(); + + IEnumerable superBundlesToAdd = Enumerable.Empty(); + if (flags.HasFlag(BaseModResource.ResourceFlags.IsAdded)) + { + // TODO: add to superbundle + } + + resources[i] = new ChunkModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, + b.Item7, Enumerable.Empty(), rangeStart, rangeEnd, logicalOffset, logicalSize, h32, + firstMip, superBundlesToAdd, Enumerable.Empty()); + break; + default: + throw new Exception("Unexpected mod resource type"); + } + } + + return retVal; + } + + private static (BaseModResource[], Block[], FrostyModDetails, uint)? UpdateLegacyFormat(DataStream inStream, + string modName) + { + DbObjectDict? mod = DbObject.Deserialize(inStream)?.AsDict(); + if (mod is null) + { + return null; + } + + return default; + } + + private static (int, string, Sha1, long, int, string, IEnumerable) ReadBaseModResource(DataStream inStream, uint version) + { + (int, string, Sha1, long, int, string, IEnumerable) retVal = default; + + retVal.Item1 = inStream.ReadInt32(); + + retVal.Item2 = (version < 4 && retVal.Item1 != -1) || version > 3 ? inStream.ReadNullTerminatedString() : string.Empty; + + if (retVal.Item1 != -1) + { + retVal.Item3 = inStream.ReadSha1(); + retVal.Item4 = inStream.ReadInt64(); + inStream.Position += 1; // we discard the flags and just check with the AssetManager if the asset was added + retVal.Item5 = inStream.ReadInt32(); + retVal.Item6 = version > 2 ? inStream.ReadNullTerminatedString() : string.Empty; + } + + // prior to version 4, mods stored bundles the asset already existed in for modification + // so must read and ignore this list + if (version < 4 && retVal.Item1 != -1) + { + int count = inStream.ReadInt32(); + inStream.Position += count * sizeof(int); + + count = inStream.ReadInt32(); + HashSet bundles = new(count); + for (int i = 0; i < count; i++) + { + int hash = inStream.ReadInt32(); + bundles.UnionWith(s_bundleMapping[hash]); + } + retVal.Item7 = bundles; + } + + // as of version 4, only bundles the asset will be added to are stored, existing bundles + // are extracted from the asset manager during the apply process + else if (version > 3) + { + int count = inStream.ReadInt32(); + HashSet bundles = new(count); + for (int i = 0; i < count; i++) + { + int hash = inStream.ReadInt32(); + bundles.UnionWith(s_bundleMapping[hash]); + } + retVal.Item7 = bundles; + } + + return retVal; + } +} \ No newline at end of file diff --git a/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs b/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs index 137d47cb1..32450ffdb 100755 --- a/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs +++ b/FrostyModSupport/Mod/Resources/EmbeddedModResource.cs @@ -12,7 +12,7 @@ public EmbeddedModResource(DataStream inStream) { } - internal EmbeddedModResource(int inResourceIndex, string inName) + public EmbeddedModResource(int inResourceIndex, string inName) : base(inResourceIndex, inName, Sha1.Zero, 0, 0, 0, string.Empty, Enumerable.Empty(), Enumerable.Empty()) { From 351935319ebc384a95ddaf6dc02ee30a6b94e3b3 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sat, 21 Oct 2023 00:06:43 +0200 Subject: [PATCH 05/14] [ModSupport] Some more stuff --- .../Attributes/HandlerAttribute.cs | 11 + FrostyModSupport/FrostyModExecutor.cs | 223 +++++++++++++++--- FrostyModSupport/Interfaces/IHandler.cs | 8 + .../Interfaces/IResourceContainer.cs | 3 +- FrostyModSupport/Mod/FrostyMod.cs | 11 +- FrostyModSupport/Mod/FrostyModCollection.cs | 107 ++++++++- FrostyModSupport/ModEntries/ChunkModEntry.cs | 29 ++- FrostyModSupport/ModEntries/EbxModEntry.cs | 12 +- FrostyModSupport/ModEntries/ResModEntry.cs | 24 +- .../ModInfos/SuperBundleModInfo.cs | 1 + 10 files changed, 386 insertions(+), 43 deletions(-) create mode 100644 FrostyModSupport/Attributes/HandlerAttribute.cs create mode 100644 FrostyModSupport/Interfaces/IHandler.cs diff --git a/FrostyModSupport/Attributes/HandlerAttribute.cs b/FrostyModSupport/Attributes/HandlerAttribute.cs new file mode 100644 index 000000000..05aff7e0a --- /dev/null +++ b/FrostyModSupport/Attributes/HandlerAttribute.cs @@ -0,0 +1,11 @@ +namespace Frosty.ModSupport.Attributes; + +public class HandlerAttribute : Attribute +{ + public int Hash { get; } + + public HandlerAttribute(int inHash) + { + Hash = inHash; + } +} \ No newline at end of file diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 6d02dc7b7..67a42cd22 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -1,9 +1,13 @@ +using System.Diagnostics; +using System.Reflection; using System.Text.Json; +using Frosty.ModSupport.Attributes; using Frosty.ModSupport.Interfaces; using Frosty.ModSupport.Mod; using Frosty.ModSupport.Mod.Resources; using Frosty.ModSupport.ModEntries; using Frosty.ModSupport.ModInfos; +using Frosty.Sdk; using Frosty.Sdk.Managers; using Frosty.Sdk.Managers.Entries; using Frosty.Sdk.Managers.Infos; @@ -16,8 +20,12 @@ public class FrostyModExecutor private Dictionary m_modifiedRes = new(); private Dictionary m_modifiedChunks = new(); + private Dictionary m_data = new(); + private Dictionary m_superBundleModInfos = new(); private Dictionary m_mapping = new(); + + private Dictionary m_handlers = new(); /// /// Generates a directory containing the modded games data. @@ -47,6 +55,8 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) ResourceManager.Initialize(); AssetManager.Initialize(); + LoadHandlers(); + // create bundle lookup map GenerateBundleLookup(); @@ -108,6 +118,27 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) return Errors.Success; } + private void LoadHandlers() + { + foreach (string handler in Directory.EnumerateFiles("Handlers")) + { + Assembly assembly = Assembly.Load(handler); + foreach (Type type in assembly.ExportedTypes) + { + if (type.IsSubclassOf(typeof(IHandler))) + { + HandlerAttribute? attribute = type.GetCustomAttribute(); + if (attribute is null) + { + continue; + } + m_handlers.TryAdd(attribute.Hash, type); + } + } + } + throw new NotImplementedException(); + } + private void GenerateBundleLookup() { foreach (SuperBundleInfo sb in FileSystemManager.EnumerateSuperBundles()) @@ -120,7 +151,6 @@ private void GenerateBundleLookup() } } } - } private void ProcessModResources(IResourceContainer container) @@ -132,53 +162,188 @@ private void ProcessModResources(IResourceContainer container) { case BundleModResource: break; - case EbxModResource: + case EbxModResource ebx: { - if (resource.IsModified || !m_modifiedEbx.ContainsKey(resource.Name)) + bool exists; + if ((exists = m_modifiedEbx.ContainsKey(resource.Name)) && !resource.HasHandler) + { + // asset was already modified by another mod so just skip to the bundle part + break; + } + + EbxModEntry modEntry; + + if (resource.HasHandler) { - if (resource.HasHandler) + if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) { - + break; } - else + + if (exists) { - if (m_modifiedEbx.TryGetValue(resource.Name, out EbxModEntry? existingEntry)) + modEntry = m_modifiedEbx[resource.Name]; + if (modEntry.Handler is null) { - if (existingEntry.Sha1 == resource.Sha1 /*|| has handler*/) - { - break; - } - - m_modifiedEbx.Remove(resource.Name, out _); + break; } + } + else + { + modEntry = new EbxModEntry(ebx, -1); + modEntry.Handler = (IHandler)Activator.CreateInstance(type)!; + } + + modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); + break; + } - // TODO: create EbxModEntry from resource - - EbxAssetEntry? ebxEntry = AssetManager.GetEbxAssetEntry(resource.Name); + EbxAssetEntry? entry = AssetManager.GetEbxAssetEntry(resource.Name); - if (resource.ResourceIndex == -1) + if (resource.IsModified) + { + // only add asset to bundles, use base games data + // TODO: get data from base game + modEntry = new EbxModEntry(ebx, -1); + } + else + { + FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + Debug.Assert(m_data.TryAdd(resource.Sha1, data)); + modEntry = new EbxModEntry(ebx, data.Size); + + if (entry is not null) + { + // add in existing bundles + foreach (int bundle in entry.Bundles) { - // only add asset to bundles, use base games data - } - else if (ebxEntry is not null) + modifiedBundles.Add(bundle); + } + } + } + + m_modifiedEbx.Add(resource.Name, modEntry); + break; + } + case ResModResource res: + { + bool exists; + if ((exists = m_modifiedRes.ContainsKey(resource.Name)) && !resource.HasHandler) + { + // asset was already modified by another mod so just skip to the bundle part + break; + } + + ResModEntry modEntry; + + if (resource.HasHandler) + { + if (exists) + { + modEntry = m_modifiedRes[resource.Name]; + if (modEntry.Handler is null) { - // add in existing bundles - foreach (int bundle in ebxEntry.Bundles) - { - modifiedBundles.Add(bundle); - } + break; } } + else + { + modEntry = new ResModEntry(res, -1); + // TODO: Get handler for asset + // modEntry.Handler = ; + } + + modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); + break; } - } - break; - case ResModResource: + + ResAssetEntry? entry = AssetManager.GetResAssetEntry(resource.Name); + + if (resource.IsModified) + { + // only add asset to bundles, use base games data + // TODO: get data from base game + modEntry = new ResModEntry(res, -1); + } + else + { + FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + Debug.Assert(m_data.TryAdd(resource.Sha1, data)); + modEntry = new ResModEntry(res, data.Size); + + if (entry is not null) + { + // add in existing bundles + foreach (int bundle in entry.Bundles) + { + modifiedBundles.Add(bundle); + } + } + } + + m_modifiedRes.Add(resource.Name, modEntry); break; + } case ChunkModResource chunk: - foreach (int superBundle in chunk.AddedSuperBundles) + { + Guid id = Guid.Parse(resource.Name); + bool exists; + if ((exists = m_modifiedChunks.ContainsKey(id)) && !resource.HasHandler) + { + // asset was already modified by another mod so just skip to the bundle part + break; + } + + ChunkModEntry modEntry; + + if (resource.HasHandler) { + if (exists) + { + modEntry = m_modifiedChunks[id]; + if (modEntry.Handler is null) + { + break; + } + } + else + { + modEntry = new ChunkModEntry(chunk, -1); + // TODO: Get handler for asset + // modEntry.Handler = ; + } + + modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); + break; } + + ChunkAssetEntry? entry = AssetManager.GetChunkAssetEntry(id); + + if (resource.IsModified) + { + // only add asset to bundles, use base games data + // TODO: get data from base game + modEntry = new ChunkModEntry(chunk, -1); + } + else + { + FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + Debug.Assert(m_data.TryAdd(resource.Sha1, data)); + modEntry = new ChunkModEntry(chunk, data.Size); + + if (entry is not null) + { + // add in existing bundles + foreach (int bundle in entry.Bundles) + { + modifiedBundles.Add(bundle); + } + } + } + + m_modifiedChunks.Add(id, modEntry); break; + } case FsFileModResource: break; } diff --git a/FrostyModSupport/Interfaces/IHandler.cs b/FrostyModSupport/Interfaces/IHandler.cs new file mode 100644 index 000000000..20e498117 --- /dev/null +++ b/FrostyModSupport/Interfaces/IHandler.cs @@ -0,0 +1,8 @@ +using Frosty.Sdk.Utils; + +namespace Frosty.ModSupport.Interfaces; + +public interface IHandler +{ + public void Load(Block inData); +} \ No newline at end of file diff --git a/FrostyModSupport/Interfaces/IResourceContainer.cs b/FrostyModSupport/Interfaces/IResourceContainer.cs index c273970ba..ad3e27869 100644 --- a/FrostyModSupport/Interfaces/IResourceContainer.cs +++ b/FrostyModSupport/Interfaces/IResourceContainer.cs @@ -1,3 +1,4 @@ +using Frosty.ModSupport.Mod; using Frosty.ModSupport.Mod.Resources; using Frosty.Sdk.Utils; @@ -15,5 +16,5 @@ public interface IResourceContainer /// /// The index of the resource. /// - public Block GetData(int inIndex); + public FrostyMod.ResourceData GetData(int inIndex); } \ No newline at end of file diff --git a/FrostyModSupport/Mod/FrostyMod.cs b/FrostyModSupport/Mod/FrostyMod.cs index 9b9fc0b99..67d37d364 100644 --- a/FrostyModSupport/Mod/FrostyMod.cs +++ b/FrostyModSupport/Mod/FrostyMod.cs @@ -11,6 +11,8 @@ public class FrostyMod : IResourceContainer { public class ResourceData { + public long Size => m_size; + private string m_fileName; private long m_offset; private int m_size; @@ -69,9 +71,9 @@ private FrostyMod(FrostyModDetails inModDetails, uint inHead, BaseModResource[] m_data = inData; } - public Block GetData(int inIndex) + public ResourceData GetData(int inIndex) { - return m_data[inIndex].GetData(); + return m_data[inIndex]; } /// @@ -183,10 +185,7 @@ public Block GetData(int inIndex) return null; } - if (FileSystemManager.Head != stream.ReadUInt32()) - { - // made for a different version of the game, may or may not work - } + uint head = stream.ReadUInt32(); return new FrostyModDetails(stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), stream.ReadNullTerminatedString(), diff --git a/FrostyModSupport/Mod/FrostyModCollection.cs b/FrostyModSupport/Mod/FrostyModCollection.cs index 47d55fd88..8eaf1b4c8 100644 --- a/FrostyModSupport/Mod/FrostyModCollection.cs +++ b/FrostyModSupport/Mod/FrostyModCollection.cs @@ -1,16 +1,115 @@ +using System.Text.Json; +using Frosty.Sdk.IO; + namespace Frosty.ModSupport.Mod; public class FrostyModCollection { - public IEnumerable Mods { get; } + public class Manifest + { + public string Link { get; set; } = string.Empty; + public string Title { get; set; } = string.Empty; + public string Author { get; set; } = string.Empty; + public string Version { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; + public string Category { get; set; } = string.Empty; + public List Mods { get; set; } = new(); + public List ModVersions { get; set; } = new(); + } + + public IEnumerable Mods => m_mods; + + /// + /// 'FCOL' + /// + private static readonly uint s_magic = 0x46434F4C; + private static readonly uint s_version = 1; + + private FrostyMod.ResourceData m_icon; + private FrostyMod.ResourceData[] m_screenshots; + private FrostyMod[] m_mods; public static FrostyModCollection? Load(string inPath) { - throw new NotImplementedException(); + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + if (s_magic != stream.ReadUInt32()) + { + return null; + } + + if (s_version != stream.ReadUInt32()) + { + return null; + } + + uint manifestOffset = stream.ReadUInt32(); + int manifestSize = stream.ReadInt32(); + FrostyMod.ResourceData icon = new(inPath, stream.ReadUInt32(), stream.ReadInt32()); + uint screenShotsOffset = stream.ReadUInt32(); + + stream.Position = manifestOffset; + Span utf8Json = new byte[manifestSize]; + stream.ReadExactly(utf8Json); + Manifest? manifest = JsonSerializer.Deserialize(utf8Json); + if (manifest is null) + { + // should never happen, since invalid or corrupted mods should be caught with magic and version + return null; + } + + FrostyModDetails modDetails = new(manifest.Title, manifest.Author, manifest.Category, manifest.Version, + manifest.Description, manifest.Link); + + stream.Position = screenShotsOffset; + int count = stream.ReadInt32(); + FrostyMod.ResourceData[] screenshots = new FrostyMod.ResourceData[count]; + long offset = screenShotsOffset + 4 + 4; + for (int i = 0; i < count; i++) + { + int size = stream.ReadInt32(); + screenshots[i] = new FrostyMod.ResourceData(inPath, offset, size); + offset += size + 4; + } + + FrostyMod[] mods = new FrostyMod[manifest.Mods.Count]; + for (int i = 0; i < mods.Length; i++) + { + } + } + + return default; } - public static FrostyModDetails GetModDetails(string inPath) + public static FrostyModDetails? GetModDetails(string inPath) { - throw new NotImplementedException(); + using (BlockStream stream = BlockStream.FromFile(inPath, false)) + { + if (s_magic != stream.ReadUInt32()) + { + return null; + } + + if (s_version != stream.ReadUInt32()) + { + return null; + } + + uint manifestOffset = stream.ReadUInt32(); + int manifestSize = stream.ReadInt32(); + + stream.Position = manifestOffset; + Span utf8Json = new byte[manifestSize]; + stream.ReadExactly(utf8Json); + Manifest? manifest = JsonSerializer.Deserialize(utf8Json); + + if (manifest is null) + { + return null; + } + + return new FrostyModDetails(manifest.Title, manifest.Author, manifest.Category, + manifest.Version, manifest.Description, manifest.Link); + } } } \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ChunkModEntry.cs b/FrostyModSupport/ModEntries/ChunkModEntry.cs index 3a837d139..ef4876a40 100644 --- a/FrostyModSupport/ModEntries/ChunkModEntry.cs +++ b/FrostyModSupport/ModEntries/ChunkModEntry.cs @@ -1,6 +1,33 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod.Resources; +using Frosty.Sdk; + namespace Frosty.ModSupport.ModEntries; public class ChunkModEntry { - + public Guid Id { get; } + public Sha1 Sha1 { get; } + public uint RangeStart { get; } + public uint RangeEnd { get; } + public uint LogicalOffset { get; } + public uint LogicalSize { get; } + public int H32 { get; } + public int FirstMip { get; } + public long Size { get; } + public IHandler? Handler { get; set; } + + public ChunkModEntry(ChunkModResource inResource, long inSize) + { + Id = Guid.Parse(inResource.Name); + Sha1 = inResource.Sha1; + RangeStart = inResource.RangeStart; + RangeEnd = inResource.RangeEnd; + LogicalOffset = inResource.LogicalOffset; + LogicalSize = inResource.LogicalSize; + H32 = inResource.H32; + FirstMip = inResource.FirstMip; + Size = inSize; + + } } \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/EbxModEntry.cs b/FrostyModSupport/ModEntries/EbxModEntry.cs index b94cebc5a..5dbeaed42 100644 --- a/FrostyModSupport/ModEntries/EbxModEntry.cs +++ b/FrostyModSupport/ModEntries/EbxModEntry.cs @@ -1,3 +1,5 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod.Resources; using Frosty.Sdk; namespace Frosty.ModSupport.ModEntries; @@ -8,5 +10,13 @@ public class EbxModEntry public Sha1 Sha1 { get; } public long OriginalSize { get; } public long Size { get; } - + public IHandler? Handler { get; set; } + + public EbxModEntry(EbxModResource inResource, long inSize) + { + Name = inResource.Name; + Sha1 = inResource.Sha1; + OriginalSize = inResource.OriginalSize; + Size = inSize; + } } \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ResModEntry.cs b/FrostyModSupport/ModEntries/ResModEntry.cs index 7356d3996..be4c8359d 100644 --- a/FrostyModSupport/ModEntries/ResModEntry.cs +++ b/FrostyModSupport/ModEntries/ResModEntry.cs @@ -1,6 +1,28 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.ModSupport.Mod.Resources; +using Frosty.Sdk; + namespace Frosty.ModSupport.ModEntries; public class ResModEntry { - + public string Name { get; } + public Sha1 Sha1 { get; } + public long OriginalSize { get; } + public ulong ResRid { get; } + public uint ResType { get; } + public byte[] ResMeta { get; } + public long Size { get; } + public IHandler? Handler { get; set; } + + public ResModEntry(ResModResource inResource, long inSize) + { + Name = inResource.Name; + Sha1 = inResource.Sha1; + OriginalSize = inResource.OriginalSize; + ResRid = inResource.ResRid; + ResType = inResource.ResType; + ResMeta = inResource.ResMeta; + Size = inSize; + } } \ No newline at end of file diff --git a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs index a67bb326c..ce102cb0c 100644 --- a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs +++ b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs @@ -5,4 +5,5 @@ public class SuperBundleModInfo public SuperBundleModAction Added = new(); public SuperBundleModAction Removed = new(); public SuperBundleModAction Modified = new(); + public string Name; } \ No newline at end of file From c4b1e58df4a1100991e929665ad5ecb833b012dd Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sat, 21 Oct 2023 13:24:45 +0200 Subject: [PATCH 06/14] [ModSupport] Update ResourceData namespace --- FrostyModSupport/FrostyModExecutor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 67a42cd22..12236773b 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -20,7 +20,7 @@ public class FrostyModExecutor private Dictionary m_modifiedRes = new(); private Dictionary m_modifiedChunks = new(); - private Dictionary m_data = new(); + private Dictionary m_data = new(); private Dictionary m_superBundleModInfos = new(); private Dictionary m_mapping = new(); @@ -208,7 +208,7 @@ private void ProcessModResources(IResourceContainer container) } else { - FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + ResourceData data = container.GetData(resource.ResourceIndex); Debug.Assert(m_data.TryAdd(resource.Sha1, data)); modEntry = new EbxModEntry(ebx, data.Size); @@ -267,7 +267,7 @@ private void ProcessModResources(IResourceContainer container) } else { - FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + ResourceData data = container.GetData(resource.ResourceIndex); Debug.Assert(m_data.TryAdd(resource.Sha1, data)); modEntry = new ResModEntry(res, data.Size); @@ -327,7 +327,7 @@ private void ProcessModResources(IResourceContainer container) } else { - FrostyMod.ResourceData data = container.GetData(resource.ResourceIndex); + ResourceData data = container.GetData(resource.ResourceIndex); Debug.Assert(m_data.TryAdd(resource.Sha1, data)); modEntry = new ChunkModEntry(chunk, data.Size); From 8184ecf25b76bf3e9e66874a56fd212bfea90996 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sat, 21 Oct 2023 13:29:04 +0200 Subject: [PATCH 07/14] [ModSupport] Move ModUpdater to own branch --- FrostyModSupport/Mod/ModUpdater.cs | 203 ----------------------------- 1 file changed, 203 deletions(-) delete mode 100644 FrostyModSupport/Mod/ModUpdater.cs diff --git a/FrostyModSupport/Mod/ModUpdater.cs b/FrostyModSupport/Mod/ModUpdater.cs deleted file mode 100644 index 408d166e6..000000000 --- a/FrostyModSupport/Mod/ModUpdater.cs +++ /dev/null @@ -1,203 +0,0 @@ -using Frosty.ModSupport.Mod.Resources; -using Frosty.Sdk; -using Frosty.Sdk.DbObjectElements; -using Frosty.Sdk.IO; -using Frosty.Sdk.Managers; -using Frosty.Sdk.Utils; - -namespace Frosty.ModSupport.Mod; - -public class ModUpdater -{ - private static Dictionary> s_bundleMapping = new(); - - public static Errors UpdateMod(string inPath) - { - (BaseModResource[], Block[], FrostyModDetails, uint)? mod; - - string extension = Path.GetExtension(inPath); - - if (extension == ".daimod") - { - // TODO: daimod convert - } - else if (extension != ".fbmod") - { - return Errors.InvalidMods; - } - - using (BlockStream stream = BlockStream.FromFile(inPath, false)) - { - // read header - if (FrostyMod.Magic != stream.ReadUInt64()) - { - stream.Position = 0; - mod = UpdateLegacyFormat(stream, inPath.Replace(".fbmod", string.Empty)); - } - else - { - mod = UpdateNewFormat(stream); - } - } - - if (mod is null) - { - return Errors.InvalidMods; - } - - FrostyMod.Save(inPath, mod.Value.Item1, mod.Value.Item2, mod.Value.Item3, mod.Value.Item4); - - foreach (Block block in mod.Value.Item2) - { - block.Dispose(); - } - - return Errors.Success; - } - - private static (BaseModResource[], Block[], FrostyModDetails, uint)? UpdateNewFormat(DataStream inStream) - { - uint version = inStream.ReadUInt32(); - - long dataOffset = inStream.ReadInt64(); - int dataCount = inStream.ReadInt32(); - - if (ProfilesLibrary.ProfileName.Equals(inStream.ReadSizedString(), StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - (BaseModResource[], Block[], FrostyModDetails, uint) retVal = default; - retVal.Item4 = inStream.ReadUInt32(); - - retVal.Item3 = new FrostyModDetails(inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), - inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), inStream.ReadNullTerminatedString(), - version > 4 ? inStream.ReadNullTerminatedString() : string.Empty); - - int resourceCount = inStream.ReadInt32(); - BaseModResource[] resources = new BaseModResource[resourceCount]; - for (int i = 0; i < resourceCount; i++) - { - ModResourceType type = (ModResourceType)inStream.ReadByte(); - (int, string, Sha1, long, int, string, IEnumerable) b; - BaseModResource.ResourceFlags flags; - switch (type) - { - case ModResourceType.Embedded: - b = ReadBaseModResource(inStream, version); - resources[i] = new EmbeddedModResource(b.Item1, b.Item2); - break; - case ModResourceType.Bundle: - b = ReadBaseModResource(inStream, version); - inStream.ReadNullTerminatedString(); - int superBundleHash = inStream.ReadInt32(); - // TODO: update hash and add bundle hash to s_bundleMapping - - resources[i] = new BundleModResource(b.Item2, superBundleHash); - break; - case ModResourceType.Ebx: - b = ReadBaseModResource(inStream, version); - flags = AssetManager.GetEbxAssetEntry(b.Item2) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; - - resources[i] = new EbxModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, b.Item7, Enumerable.Empty()); - break; - case ModResourceType.Res: - b = ReadBaseModResource(inStream, version); - flags = AssetManager.GetResAssetEntry(b.Item2) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; - - resources[i] = new ResModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, b.Item7, - Enumerable.Empty(), inStream.ReadUInt32(), inStream.ReadUInt64(), - inStream.ReadBytes(inStream.ReadInt32())); - break; - case ModResourceType.Chunk: - b = ReadBaseModResource(inStream, version); - flags = AssetManager.GetChunkAssetEntry(Guid.Parse(b.Item2)) is not null ? BaseModResource.ResourceFlags.IsAdded : 0; - - uint rangeStart = inStream.ReadUInt32(); - uint rangeEnd = inStream.ReadUInt32(); - uint logicalOffset = inStream.ReadUInt32(); - uint logicalSize = inStream.ReadUInt32(); - int h32 = inStream.ReadInt32(); - int firstMip = inStream.ReadInt32(); - - IEnumerable superBundlesToAdd = Enumerable.Empty(); - if (flags.HasFlag(BaseModResource.ResourceFlags.IsAdded)) - { - // TODO: add to superbundle - } - - resources[i] = new ChunkModResource(b.Item1, b.Item2, b.Item3, b.Item4, flags, b.Item5, b.Item6, - b.Item7, Enumerable.Empty(), rangeStart, rangeEnd, logicalOffset, logicalSize, h32, - firstMip, superBundlesToAdd, Enumerable.Empty()); - break; - default: - throw new Exception("Unexpected mod resource type"); - } - } - - return retVal; - } - - private static (BaseModResource[], Block[], FrostyModDetails, uint)? UpdateLegacyFormat(DataStream inStream, - string modName) - { - DbObjectDict? mod = DbObject.Deserialize(inStream)?.AsDict(); - if (mod is null) - { - return null; - } - - return default; - } - - private static (int, string, Sha1, long, int, string, IEnumerable) ReadBaseModResource(DataStream inStream, uint version) - { - (int, string, Sha1, long, int, string, IEnumerable) retVal = default; - - retVal.Item1 = inStream.ReadInt32(); - - retVal.Item2 = (version < 4 && retVal.Item1 != -1) || version > 3 ? inStream.ReadNullTerminatedString() : string.Empty; - - if (retVal.Item1 != -1) - { - retVal.Item3 = inStream.ReadSha1(); - retVal.Item4 = inStream.ReadInt64(); - inStream.Position += 1; // we discard the flags and just check with the AssetManager if the asset was added - retVal.Item5 = inStream.ReadInt32(); - retVal.Item6 = version > 2 ? inStream.ReadNullTerminatedString() : string.Empty; - } - - // prior to version 4, mods stored bundles the asset already existed in for modification - // so must read and ignore this list - if (version < 4 && retVal.Item1 != -1) - { - int count = inStream.ReadInt32(); - inStream.Position += count * sizeof(int); - - count = inStream.ReadInt32(); - HashSet bundles = new(count); - for (int i = 0; i < count; i++) - { - int hash = inStream.ReadInt32(); - bundles.UnionWith(s_bundleMapping[hash]); - } - retVal.Item7 = bundles; - } - - // as of version 4, only bundles the asset will be added to are stored, existing bundles - // are extracted from the asset manager during the apply process - else if (version > 3) - { - int count = inStream.ReadInt32(); - HashSet bundles = new(count); - for (int i = 0; i < count; i++) - { - int hash = inStream.ReadInt32(); - bundles.UnionWith(s_bundleMapping[hash]); - } - retVal.Item7 = bundles; - } - - return retVal; - } -} \ No newline at end of file From d7132a567e21ec2e43fa9fc4ded6e36baf073916 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sun, 22 Oct 2023 22:33:11 +0200 Subject: [PATCH 08/14] [Sdk] Update methods to get bundles and sbic --- FrostySdk/Managers/AssetManager.cs | 10 +++++----- FrostySdk/Managers/FileSystemManager.cs | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/FrostySdk/Managers/AssetManager.cs b/FrostySdk/Managers/AssetManager.cs index 39514f457..46dd3b03a 100644 --- a/FrostySdk/Managers/AssetManager.cs +++ b/FrostySdk/Managers/AssetManager.cs @@ -195,13 +195,13 @@ public static bool Initialize(PatchResult? patchResult = null) #region -- Bundle -- /// - /// Gets the by name. + /// Gets the by hash. /// - /// The Id of the Bundle. - /// The or null if it doesn't exist. - public static BundleInfo? GetBundleEntry(int id) + /// The hash of the Bundle. + /// The or null if it doesn't exist. + public static BundleInfo? GetBundleInfo(int inHash) { - return s_bundleMapping.Count > id ? s_bundleMapping[id] : null; + return s_bundleMapping.TryGetValue(inHash, out BundleInfo? bundleInfo) ? bundleInfo : null; } #endregion diff --git a/FrostySdk/Managers/FileSystemManager.cs b/FrostySdk/Managers/FileSystemManager.cs index 4a0640b60..968ea6825 100644 --- a/FrostySdk/Managers/FileSystemManager.cs +++ b/FrostySdk/Managers/FileSystemManager.cs @@ -194,6 +194,11 @@ public static SuperBundleInstallChunk GetSuperBundleInstallChunk(string sbIcName { return s_sbIcMapping[Utils.Utils.HashString(sbIcName, true)]; } + + public static SuperBundleInstallChunk GetSuperBundleInstallChunk(int hash) + { + return s_sbIcMapping[hash]; + } public static bool HasFileInMemoryFs(string name) => s_memoryFs.ContainsKey(name); public static Block GetFileFromMemoryFs(string name) => s_memoryFs[name]; From 4f99de4b4f248a7b6e33fd84e573aa7d5f02b522 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sun, 22 Oct 2023 22:37:35 +0200 Subject: [PATCH 09/14] [ModSupport] Fixed some issues with bundle and superbundles --- FrostyModSupport/FrostyModExecutor.cs | 139 +++++++++++------- .../ModInfos/SuperBundleModInfo.cs | 1 - 2 files changed, 89 insertions(+), 51 deletions(-) diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 12236773b..698a1d677 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -11,6 +11,7 @@ using Frosty.Sdk.Managers; using Frosty.Sdk.Managers.Entries; using Frosty.Sdk.Managers.Infos; +using Frosty.Sdk.Utils; namespace Frosty.ModSupport; @@ -22,8 +23,8 @@ public class FrostyModExecutor private Dictionary m_data = new(); - private Dictionary m_superBundleModInfos = new(); - private Dictionary m_mapping = new(); + private Dictionary m_superBundleModInfos = new(); + private Dictionary m_bundleToSuperBundleMapping = new(); private Dictionary m_handlers = new(); @@ -56,9 +57,6 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) AssetManager.Initialize(); LoadHandlers(); - - // create bundle lookup map - GenerateBundleLookup(); // process all mods foreach (string path in modPaths) @@ -100,7 +98,7 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) } } - foreach (KeyValuePair sb in m_superBundleModInfos) + foreach (SuperBundleModInfo sb in m_superBundleModInfos.Values) { switch (FileSystemManager.BundleFormat) { @@ -139,20 +137,6 @@ private void LoadHandlers() throw new NotImplementedException(); } - private void GenerateBundleLookup() - { - foreach (SuperBundleInfo sb in FileSystemManager.EnumerateSuperBundles()) - { - foreach (SuperBundleInstallChunk sbIc in sb.InstallChunks) - { - foreach (KeyValuePair bundle in sbIc.BundleMapping) - { - m_mapping.Add(bundle.Value.Id, sbIc.Name); - } - } - } - } - private void ProcessModResources(IResourceContainer container) { foreach (BaseModResource resource in container.Resources) @@ -160,8 +144,15 @@ private void ProcessModResources(IResourceContainer container) HashSet modifiedBundles = new(); switch (resource) { - case BundleModResource: + case BundleModResource bundle: + { + SuperBundleModInfo sb = GetSuperBundleModInfo(bundle.SuperBundleHash); + + int bundleHash = Utils.HashString(bundle.Name + FileSystemManager.GetSuperBundleInstallChunk(bundle.SuperBundleHash).Name, true); + sb.Added.Bundles.TryAdd(bundleHash, new BundleModInfo()); + m_bundleToSuperBundleMapping.TryAdd(bundleHash, bundle.SuperBundleHash); break; + } case EbxModResource ebx: { bool exists; @@ -190,8 +181,10 @@ private void ProcessModResources(IResourceContainer container) } else { - modEntry = new EbxModEntry(ebx, -1); - modEntry.Handler = (IHandler)Activator.CreateInstance(type)!; + modEntry = new EbxModEntry(ebx, -1) + { + Handler = (IHandler)Activator.CreateInstance(type)! + }; } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -238,6 +231,11 @@ private void ProcessModResources(IResourceContainer container) if (resource.HasHandler) { + if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) + { + break; + } + if (exists) { modEntry = m_modifiedRes[resource.Name]; @@ -248,9 +246,10 @@ private void ProcessModResources(IResourceContainer container) } else { - modEntry = new ResModEntry(res, -1); - // TODO: Get handler for asset - // modEntry.Handler = ; + modEntry = new ResModEntry(res, -1) + { + Handler = (IHandler)Activator.CreateInstance(type)! + }; } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -298,6 +297,11 @@ private void ProcessModResources(IResourceContainer container) if (resource.HasHandler) { + if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) + { + break; + } + if (exists) { modEntry = m_modifiedChunks[id]; @@ -308,9 +312,10 @@ private void ProcessModResources(IResourceContainer container) } else { - modEntry = new ChunkModEntry(chunk, -1); - // TODO: Get handler for asset - // modEntry.Handler = ; + modEntry = new ChunkModEntry(chunk, -1) + { + Handler = (IHandler)Activator.CreateInstance(type)! + }; } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -337,9 +342,27 @@ private void ProcessModResources(IResourceContainer container) foreach (int bundle in entry.Bundles) { modifiedBundles.Add(bundle); - } + } + + foreach (int superBundle in entry.SuperBundleInstallChunks) + { + SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); + sb.Modified.Chunks.Add(id); + } } } + + foreach (int superBundle in chunk.AddedSuperBundles) + { + SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); + sb.Added.Chunks.Add(id); + } + + foreach (int superBundle in chunk.RemovedSuperBundles) + { + SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); + sb.Removed.Chunks.Add(id); + } m_modifiedChunks.Add(id, modEntry); break; @@ -350,12 +373,7 @@ private void ProcessModResources(IResourceContainer container) foreach (int addedBundle in resource.AddedBundles) { - string sbName = m_mapping[addedBundle]; - if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) - { - sb = new SuperBundleModInfo(); - m_superBundleModInfos.Add(sbName, sb); - } + SuperBundleModInfo sb = GetSuperBundleModInfoFromBundle(addedBundle); if (!sb.Modified.Bundles.TryGetValue(addedBundle, out BundleModInfo? modInfo)) { @@ -379,12 +397,7 @@ private void ProcessModResources(IResourceContainer container) foreach (int removedBundle in resource.RemovedBundles) { - string sbName = m_mapping[removedBundle]; - if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) - { - sb = new SuperBundleModInfo(); - m_superBundleModInfos.Add(sbName, sb); - } + SuperBundleModInfo sb = GetSuperBundleModInfoFromBundle(removedBundle); if (!sb.Modified.Bundles.TryGetValue(removedBundle, out BundleModInfo? modInfo)) { @@ -408,13 +421,8 @@ private void ProcessModResources(IResourceContainer container) foreach (int modifiedBundle in modifiedBundles) { - string sbName = m_mapping[modifiedBundle]; - if (!m_superBundleModInfos.TryGetValue(sbName, out SuperBundleModInfo? sb)) - { - sb = new SuperBundleModInfo(); - m_superBundleModInfos.Add(sbName, sb); - } - + SuperBundleModInfo sb = GetSuperBundleModInfoFromBundle(modifiedBundle); + if (!sb.Modified.Bundles.TryGetValue(modifiedBundle, out BundleModInfo? modInfo)) { modInfo = new BundleModInfo(); @@ -436,7 +444,38 @@ private void ProcessModResources(IResourceContainer container) } } } - + + private SuperBundleModInfo GetSuperBundleModInfoFromBundle(int inBundle) + { + BundleInfo? bundle = AssetManager.GetBundleInfo(inBundle); + int superBundle; + if (bundle is null) + { + if (!m_bundleToSuperBundleMapping.TryGetValue(inBundle, out superBundle)) + { + // change this to a Error at some point + throw new Exception("Asset was added to Bundle, which doesnt exist."); + } + } + else + { + superBundle = Utils.HashString(bundle.Parent.Name); + } + + return GetSuperBundleModInfo(superBundle); + } + + private SuperBundleModInfo GetSuperBundleModInfo(int superBundle) + { + if (!m_superBundleModInfos.TryGetValue(superBundle, out SuperBundleModInfo? sb)) + { + sb = new SuperBundleModInfo(); + m_superBundleModInfos.Add(superBundle, sb); + } + + return sb; + } + private static List GenerateModInfoList(IEnumerable modPaths) { List modInfoList = new(); diff --git a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs index ce102cb0c..a67bb326c 100644 --- a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs +++ b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs @@ -5,5 +5,4 @@ public class SuperBundleModInfo public SuperBundleModAction Added = new(); public SuperBundleModAction Removed = new(); public SuperBundleModAction Modified = new(); - public string Name; } \ No newline at end of file From 969eeb282b44354778f5a5986bb3e9ca90de9341 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sun, 22 Oct 2023 23:39:58 +0200 Subject: [PATCH 10/14] [ModSupport] Fixed issue with handler loading --- FrostyModSupport/FrostyModExecutor.cs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 698a1d677..89f6b237c 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -17,16 +17,16 @@ namespace Frosty.ModSupport; public class FrostyModExecutor { - private Dictionary m_modifiedEbx = new(); - private Dictionary m_modifiedRes = new(); - private Dictionary m_modifiedChunks = new(); + private readonly Dictionary m_modifiedEbx = new(); + private readonly Dictionary m_modifiedRes = new(); + private readonly Dictionary m_modifiedChunks = new(); - private Dictionary m_data = new(); + private readonly Dictionary m_data = new(); - private Dictionary m_superBundleModInfos = new(); - private Dictionary m_bundleToSuperBundleMapping = new(); + private readonly Dictionary m_superBundleModInfos = new(); + private readonly Dictionary m_bundleToSuperBundleMapping = new(); - private Dictionary m_handlers = new(); + private readonly Dictionary m_handlers = new(); /// /// Generates a directory containing the modded games data. @@ -123,7 +123,7 @@ private void LoadHandlers() Assembly assembly = Assembly.Load(handler); foreach (Type type in assembly.ExportedTypes) { - if (type.IsSubclassOf(typeof(IHandler))) + if (typeof(IHandler).IsAssignableFrom(type)) { HandlerAttribute? attribute = type.GetCustomAttribute(); if (attribute is null) @@ -134,7 +134,6 @@ private void LoadHandlers() } } } - throw new NotImplementedException(); } private void ProcessModResources(IResourceContainer container) @@ -368,7 +367,10 @@ private void ProcessModResources(IResourceContainer container) break; } case FsFileModResource: + { + // TODO: break; + } } foreach (int addedBundle in resource.AddedBundles) From 81272e892a055f375ebb8af43549710a449a4b72 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:14:42 +0200 Subject: [PATCH 11/14] [ModSupport] Fixed some small issues --- FrostyModSupport/FrostyModExecutor.cs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 89f6b237c..1b0fa6e64 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -92,10 +92,6 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) ProcessModResources(mod); } } - else - { - return Errors.InvalidMods; - } } foreach (SuperBundleModInfo sb in m_superBundleModInfos.Values) @@ -167,7 +163,7 @@ private void ProcessModResources(IResourceContainer container) { if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) { - break; + continue; } if (exists) @@ -184,6 +180,7 @@ private void ProcessModResources(IResourceContainer container) { Handler = (IHandler)Activator.CreateInstance(type)! }; + m_modifiedEbx.Add(resource.Name, modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -213,7 +210,6 @@ private void ProcessModResources(IResourceContainer container) } } } - m_modifiedEbx.Add(resource.Name, modEntry); break; } @@ -232,7 +228,7 @@ private void ProcessModResources(IResourceContainer container) { if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) { - break; + continue; } if (exists) @@ -249,6 +245,7 @@ private void ProcessModResources(IResourceContainer container) { Handler = (IHandler)Activator.CreateInstance(type)! }; + m_modifiedRes.Add(resource.Name, modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -275,10 +272,9 @@ private void ProcessModResources(IResourceContainer container) foreach (int bundle in entry.Bundles) { modifiedBundles.Add(bundle); - } + } } } - m_modifiedRes.Add(resource.Name, modEntry); break; } @@ -298,7 +294,7 @@ private void ProcessModResources(IResourceContainer container) { if (!m_handlers.TryGetValue(resource.HandlerHash, out Type? type)) { - break; + continue; } if (exists) @@ -315,6 +311,7 @@ private void ProcessModResources(IResourceContainer container) { Handler = (IHandler)Activator.CreateInstance(type)! }; + m_modifiedChunks.Add(id, modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -362,7 +359,6 @@ private void ProcessModResources(IResourceContainer container) SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); sb.Removed.Chunks.Add(id); } - m_modifiedChunks.Add(id, modEntry); break; } @@ -497,7 +493,7 @@ private static List GenerateModInfoList(IEnumerable modPaths) } else { - throw new Exception(); + continue; } if (modDetails is null) From fbbd10c64e0ab77a93b3576568716da24069a2e9 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Thu, 26 Oct 2023 22:26:44 +0200 Subject: [PATCH 12/14] [ModSupport] Some more handler stuff --- FrostyModSupport/FrostyModExecutor.cs | 77 +++++++++++++++++--- FrostyModSupport/Interfaces/IHandler.cs | 12 +++ FrostyModSupport/Interfaces/IModEntry.cs | 11 +++ FrostyModSupport/ModEntries/ChunkModEntry.cs | 2 +- FrostyModSupport/ModEntries/EbxModEntry.cs | 2 +- FrostyModSupport/ModEntries/ResModEntry.cs | 2 +- 6 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 FrostyModSupport/Interfaces/IModEntry.cs diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 1b0fa6e64..a02faf7ff 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -21,7 +21,10 @@ public class FrostyModExecutor private readonly Dictionary m_modifiedRes = new(); private readonly Dictionary m_modifiedChunks = new(); + private readonly List m_handlerAssets = new(); + private readonly Dictionary m_data = new(); + private readonly Dictionary> m_data2 = new(); private readonly Dictionary m_superBundleModInfos = new(); private readonly Dictionary m_bundleToSuperBundleMapping = new(); @@ -93,9 +96,19 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) } } } - - foreach (SuperBundleModInfo sb in m_superBundleModInfos.Values) + + // apply handlers + foreach (IModEntry entry in m_handlerAssets) + { + // entry.Handler will never be null, since the assets added to m_handlerAssets always have a handler set + entry.Handler!.Modify(entry, out Block data); + Debug.Assert(m_data2.TryAdd(entry.Sha1, data)); + } + + foreach (KeyValuePair sb in m_superBundleModInfos) { + SuperBundleInstallChunk sbic = FileSystemManager.GetSuperBundleInstallChunk(sb.Key); + switch (FileSystemManager.BundleFormat) { case BundleFormat.Dynamic2018: @@ -181,6 +194,7 @@ private void ProcessModResources(IResourceContainer container) Handler = (IHandler)Activator.CreateInstance(type)! }; m_modifiedEbx.Add(resource.Name, modEntry); + m_handlerAssets.Add(modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -189,11 +203,19 @@ private void ProcessModResources(IResourceContainer container) EbxAssetEntry? entry = AssetManager.GetEbxAssetEntry(resource.Name); - if (resource.IsModified) + if (!resource.IsModified) { + // asset needs to exist if it is not modified by the game + if (entry is null) + { + // we skip the bundle part here + continue; + } + // only add asset to bundles, use base games data - // TODO: get data from base game - modEntry = new EbxModEntry(ebx, -1); + Block data = AssetManager.GetRawAsset(entry); + Debug.Assert(m_data2.TryAdd(entry.Sha1, data)); + modEntry = new EbxModEntry(ebx, data.Size); } else { @@ -246,6 +268,7 @@ private void ProcessModResources(IResourceContainer container) Handler = (IHandler)Activator.CreateInstance(type)! }; m_modifiedRes.Add(resource.Name, modEntry); + m_handlerAssets.Add(modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -254,11 +277,19 @@ private void ProcessModResources(IResourceContainer container) ResAssetEntry? entry = AssetManager.GetResAssetEntry(resource.Name); - if (resource.IsModified) + if (!resource.IsModified) { + // asset needs to exist if it is not modified by the game + if (entry is null) + { + // we skip the bundle part here + continue; + } + // only add asset to bundles, use base games data - // TODO: get data from base game - modEntry = new ResModEntry(res, -1); + Block data = AssetManager.GetRawAsset(entry); + Debug.Assert(m_data2.TryAdd(entry.Sha1, data)); + modEntry = new ResModEntry(res, data.Size); } else { @@ -312,6 +343,7 @@ private void ProcessModResources(IResourceContainer container) Handler = (IHandler)Activator.CreateInstance(type)! }; m_modifiedChunks.Add(id, modEntry); + m_handlerAssets.Add(modEntry); } modEntry.Handler.Load(container.GetData(resource.ResourceIndex).GetData()); @@ -320,11 +352,19 @@ private void ProcessModResources(IResourceContainer container) ChunkAssetEntry? entry = AssetManager.GetChunkAssetEntry(id); - if (resource.IsModified) + if (!resource.IsModified) { + // asset needs to exist if it is not modified by the game + if (entry is null) + { + // we skip the bundle part here + continue; + } + // only add asset to bundles, use base games data - // TODO: get data from base game - modEntry = new ChunkModEntry(chunk, -1); + Block data = AssetManager.GetRawAsset(entry); + Debug.Assert(m_data2.TryAdd(entry.Sha1, data)); + modEntry = new ChunkModEntry(chunk, data.Size); } else { @@ -514,4 +554,19 @@ private static List GenerateModInfoList(IEnumerable modPaths) } return modInfoList; } + + private Block GetData(Sha1 sha1) + { + if (m_data.TryGetValue(sha1, out ResourceData? data)) + { + return data.GetData(); + } + + if (m_data2.TryGetValue(sha1, out Block? block)) + { + return block; + } + + throw new Exception(); + } } \ No newline at end of file diff --git a/FrostyModSupport/Interfaces/IHandler.cs b/FrostyModSupport/Interfaces/IHandler.cs index 20e498117..4cd6a9710 100644 --- a/FrostyModSupport/Interfaces/IHandler.cs +++ b/FrostyModSupport/Interfaces/IHandler.cs @@ -1,8 +1,20 @@ +using Frosty.ModSupport.ModEntries; using Frosty.Sdk.Utils; namespace Frosty.ModSupport.Interfaces; public interface IHandler { + /// + /// Loads the data of a resource and merges it. + /// + /// The data from of the resource from the mod. public void Load(Block inData); + + /// + /// Creates the final resource. + /// + /// The mod entry to modify. + /// The final data of the resource. + public void Modify(IModEntry modEntry, out Block data); } \ No newline at end of file diff --git a/FrostyModSupport/Interfaces/IModEntry.cs b/FrostyModSupport/Interfaces/IModEntry.cs new file mode 100644 index 000000000..2c34b6eba --- /dev/null +++ b/FrostyModSupport/Interfaces/IModEntry.cs @@ -0,0 +1,11 @@ +using Frosty.ModSupport.Interfaces; +using Frosty.Sdk; + +namespace Frosty.ModSupport.ModEntries; + +public interface IModEntry +{ + public Sha1 Sha1 { get; } + + public IHandler? Handler { get; set; } +} \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ChunkModEntry.cs b/FrostyModSupport/ModEntries/ChunkModEntry.cs index ef4876a40..9147a5919 100644 --- a/FrostyModSupport/ModEntries/ChunkModEntry.cs +++ b/FrostyModSupport/ModEntries/ChunkModEntry.cs @@ -4,7 +4,7 @@ namespace Frosty.ModSupport.ModEntries; -public class ChunkModEntry +public class ChunkModEntry : IModEntry { public Guid Id { get; } public Sha1 Sha1 { get; } diff --git a/FrostyModSupport/ModEntries/EbxModEntry.cs b/FrostyModSupport/ModEntries/EbxModEntry.cs index 5dbeaed42..51ff1e6ad 100644 --- a/FrostyModSupport/ModEntries/EbxModEntry.cs +++ b/FrostyModSupport/ModEntries/EbxModEntry.cs @@ -4,7 +4,7 @@ namespace Frosty.ModSupport.ModEntries; -public class EbxModEntry +public class EbxModEntry : IModEntry { public string Name { get; } public Sha1 Sha1 { get; } diff --git a/FrostyModSupport/ModEntries/ResModEntry.cs b/FrostyModSupport/ModEntries/ResModEntry.cs index be4c8359d..c82cf73d5 100644 --- a/FrostyModSupport/ModEntries/ResModEntry.cs +++ b/FrostyModSupport/ModEntries/ResModEntry.cs @@ -4,7 +4,7 @@ namespace Frosty.ModSupport.ModEntries; -public class ResModEntry +public class ResModEntry : IModEntry { public string Name { get; } public Sha1 Sha1 { get; } From 5e5f5fa488fe26428a9a71f5547303caf4884172 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Fri, 27 Oct 2023 21:08:01 +0200 Subject: [PATCH 13/14] [ModSupport] Some more wip stuff --- .../FrostyModExecutor.Manifest2019.cs | 19 +++++++++++++ FrostyModSupport/FrostyModExecutor.cs | 28 +++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 FrostyModSupport/FrostyModExecutor.Manifest2019.cs diff --git a/FrostyModSupport/FrostyModExecutor.Manifest2019.cs b/FrostyModSupport/FrostyModExecutor.Manifest2019.cs new file mode 100644 index 000000000..5444c4277 --- /dev/null +++ b/FrostyModSupport/FrostyModExecutor.Manifest2019.cs @@ -0,0 +1,19 @@ +using Frosty.ModSupport.ModInfos; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Managers.Infos; + +namespace Frosty.ModSupport; + +public partial class FrostyModExecutor +{ + private void ModManifest2019(SuperBundleInstallChunk inSbIc, SuperBundleModInfo inModInfo) + { + string path = FileSystemManager.ResolvePath(inSbIc.Name); + if (string.IsNullOrEmpty(path)) + { + return; + } + + + } +} \ No newline at end of file diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index a02faf7ff..46c6d67ad 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -15,7 +15,7 @@ namespace Frosty.ModSupport; -public class FrostyModExecutor +public partial class FrostyModExecutor { private readonly Dictionary m_modifiedEbx = new(); private readonly Dictionary m_modifiedRes = new(); @@ -38,13 +38,15 @@ public class FrostyModExecutor /// The full paths of the mods. public Errors GenerateMods(string modPackName, params string[] modPaths) { - string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName); + // define some paths we are going to need string patchPath = FileSystemManager.Sources.Count == 1 ? FileSystemSource.Base.Path : FileSystemSource.Patch.Path; + string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName, patchPath); + string gamePatchPath = Path.Combine(FileSystemManager.BasePath, patchPath); // check if we need to generate new data - string modInfosPath = Path.Combine(modDataPath, patchPath, "mods.json"); + string modInfosPath = Path.Combine(modDataPath, "mods.json"); List modInfos = GenerateModInfoList(modPaths); if (File.Exists(modInfosPath)) { @@ -59,6 +61,7 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) ResourceManager.Initialize(); AssetManager.Initialize(); + // load handlers from Handlers directory LoadHandlers(); // process all mods @@ -105,10 +108,16 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) Debug.Assert(m_data2.TryAdd(entry.Sha1, data)); } + // clear old generated mod data + Directory.Delete(modDataPath, true); + Directory.CreateDirectory(modDataPath); + + // modify the superbundles and write them to mod data foreach (KeyValuePair sb in m_superBundleModInfos) { SuperBundleInstallChunk sbic = FileSystemManager.GetSuperBundleInstallChunk(sb.Key); + // TODO: we need to write the data to cas files (should we write them per bundle or per superbundle) and store the references, so we can write them into the cat files/directly in the bundle switch (FileSystemManager.BundleFormat) { case BundleFormat.Dynamic2018: @@ -121,6 +130,17 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) break; } } + + // create symbolic links for everything that is in gamePatchPath but not in modDataPath + foreach (string file in Directory.EnumerateFiles(gamePatchPath, string.Empty, SearchOption.AllDirectories)) + { + string modPath = Path.Combine(modDataPath, Path.GetRelativePath(gamePatchPath, file)); + if (!File.Exists(modPath)) + { + // TODO: check if we need to create the directory first + File.CreateSymbolicLink(modPath, file); + } + } return Errors.Success; } @@ -407,6 +427,8 @@ private void ProcessModResources(IResourceContainer container) // TODO: break; } + default: + continue; } foreach (int addedBundle in resource.AddedBundles) From b8043c1d73f1874077fdbbc2c36cacf1f6079991 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:09:34 +0100 Subject: [PATCH 14/14] [ModSupport] Superbundle action stuff --- .../Archive/InstallChunkWriter.cs | 74 ++++ FrostyModSupport/BinaryBundle.cs | 273 ++++++++++++++ .../FrostyModExecutor.Manifest2019.cs | 333 +++++++++++++++++- FrostyModSupport/FrostyModExecutor.cs | 65 +++- FrostyModSupport/ModEntries/ChunkModEntry.cs | 8 + FrostyModSupport/ModEntries/EbxModEntry.cs | 7 + FrostyModSupport/ModEntries/ResModEntry.cs | 11 + .../ModInfos/SuperBundleModInfo.cs | 5 +- FrostySdk/IO/BinaryBundle.cs | 6 +- FrostySdk/Managers/Infos/CasFileIdentifier.cs | 5 + .../Loaders/Manifest2019AssetLoader.cs | 4 +- 11 files changed, 768 insertions(+), 23 deletions(-) create mode 100644 FrostyModSupport/Archive/InstallChunkWriter.cs create mode 100644 FrostyModSupport/BinaryBundle.cs diff --git a/FrostyModSupport/Archive/InstallChunkWriter.cs b/FrostyModSupport/Archive/InstallChunkWriter.cs new file mode 100644 index 000000000..e242302d5 --- /dev/null +++ b/FrostyModSupport/Archive/InstallChunkWriter.cs @@ -0,0 +1,74 @@ +using Frosty.Sdk; +using Frosty.Sdk.IO; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Managers.Infos; +using Frosty.Sdk.Utils; + +namespace Frosty.ModSupport.Archive; + +public class InstallChunkWriter +{ + private InstallChunkInfo m_installChunk; + private int m_installChunkIndex; + private int m_casIndex; + private string m_dir; + private Dictionary m_data = new(); + + public InstallChunkWriter(InstallChunkInfo inInstallChunk, string inGamePatchPath, string inModDataPath) + { + m_installChunk = inInstallChunk; + m_installChunkIndex = FileSystemManager.GetInstallChunkIndex(m_installChunk); + + // get current cas index so we can use all the current cas files and dont have to rewrite the whole game + string dir = Path.Combine(inGamePatchPath, m_installChunk.InstallBundle); + if (Directory.Exists(dir)) + { + foreach (string file in Directory.EnumerateFiles(dir, "*.cas")) + { + m_casIndex = Math.Max(int.Parse(file.AsSpan()[4..][..^4]), m_casIndex); + } + } + + // create mod dir + m_dir = Path.Combine(inModDataPath, m_installChunk.InstallBundle); + Directory.CreateDirectory(m_dir); + } + + public (CasFileIdentifier, uint, uint) WriteData(Sha1 inSha1, Block inData) + { + if (m_data.TryGetValue(inSha1, out (CasFileIdentifier, uint, uint) retVal)) + { + return retVal; + } + + DataStream stream = GetCurrentWriter(inData.Size); + + if (ProfilesLibrary.FrostbiteVersion <= "2014.4.11") + { + // write faceoff header + stream.WriteUInt32(0xFACE0FF, Endian.Big); + stream.WriteSha1(inSha1); + stream.WriteInt64(inData.Size); + } + + retVal = (new CasFileIdentifier(false, m_installChunkIndex, m_casIndex), (uint)stream.Position, + (uint)inData.Size); + m_data.Add(inSha1, retVal); + stream.Write(inData); + return retVal; + } + + public (CasFileIdentifier, uint, uint) GetFileInfo(Sha1 inSha1) + { + return m_data[inSha1]; + } + + public void WriteCatalog() + { + } + + private DataStream GetCurrentWriter(int size) + { + return default; + } +} \ No newline at end of file diff --git a/FrostyModSupport/BinaryBundle.cs b/FrostyModSupport/BinaryBundle.cs new file mode 100644 index 000000000..e96f6f13c --- /dev/null +++ b/FrostyModSupport/BinaryBundle.cs @@ -0,0 +1,273 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using Frosty.ModSupport.ModEntries; +using Frosty.ModSupport.ModInfos; +using Frosty.Sdk; +using Frosty.Sdk.DbObjectElements; +using Frosty.Sdk.Exceptions; +using Frosty.Sdk.IO; +using Frosty.Sdk.Managers; +using Frosty.Sdk.Utils; + +namespace Frosty.ModSupport; + +public static class BinaryBundle +{ + public static Block Modify(BlockStream inStream, BundleModInfo inModInfo, Dictionary inModifiedEbx, Dictionary inModifiedRes, Dictionary inModifiedChunks, Action Modify) + { + // we use big endian for default + Endian endian = Endian.Big; + + uint size = inStream.ReadUInt32(Endian.Big); + + long startPos = inStream.Position; + + Sdk.IO.BinaryBundle.Magic magic = (Sdk.IO.BinaryBundle.Magic)(inStream.ReadUInt32(endian) ^ Sdk.IO.BinaryBundle.GetSalt()); + + bool containsSha1 = magic == Sdk.IO.BinaryBundle.Magic.Standard; + + uint totalCount = inStream.ReadUInt32(endian); + int ebxCount = inStream.ReadInt32(endian); + int resCount = inStream.ReadInt32(endian); + int chunkCount = inStream.ReadInt32(endian); + long stringsOffset = inStream.ReadUInt32(endian) + startPos; + long metaOffset = inStream.ReadUInt32(endian) + startPos; + inStream.Position += sizeof(int); // metaSize + + // decrypt the data + if (magic == Sdk.IO.BinaryBundle.Magic.Encrypted) + { + if (!KeyManager.HasKey("BundleEncryptionKey")) + { + throw new MissingEncryptionKeyException("bundles"); + } + + inStream.Decrypt(KeyManager.GetKey("BundleEncryptionKey"), (int)(size - 0x20), PaddingMode.None); + } + + EbxModEntry[] ebx = new EbxModEntry[ebxCount + inModInfo.Added.Ebx.Count]; + ResModEntry[] res = new ResModEntry[resCount + inModInfo.Added.Res.Count]; + ChunkModEntry[] chunks = new ChunkModEntry[chunkCount + inModInfo.Added.Chunks.Count]; + + uint offset = 0; + Dictionary strings = new(ebx.Length + res.Length); + + // read sha1s + Sha1[] sha1 = new Sha1[totalCount + inModInfo.Added.Ebx.Count + inModInfo.Added.Res.Count + inModInfo.Added.Chunks.Count]; + int b = 0; + for (int i = 0; i < totalCount; i++) + { + if (i == ebxCount) + { + b += inModInfo.Added.Ebx.Count; + } + else if (i == ebxCount + resCount) + { + b += inModInfo.Added.Res.Count; + } + sha1[i + b] = containsSha1 ? inStream.ReadSha1() : Sha1.Zero; + } + + int j = 0; + for (int i = 0; i < ebxCount; i++, j++) + { + uint nameOffset = inStream.ReadUInt32(endian); + uint originalSize = inStream.ReadUInt32(endian); + + long currentPos = inStream.Position; + inStream.Position = stringsOffset + nameOffset; + string name = inStream.ReadNullTerminatedString(); + + if (inModInfo.Modified.Ebx.Contains(name)) + { + EbxModEntry modEntry = inModifiedEbx[name]; + ebx[i] = modEntry; + Modify(modEntry, j, false); + } + else + { + ebx[i] = new EbxModEntry(name, sha1[j], originalSize); + } + + inStream.Position = currentPos; + + if (strings.TryAdd(name, offset)) + { + offset += (uint)name.Length + 1; + } + } + + int k = ebxCount; + foreach (string name in inModInfo.Added.Ebx) + { + EbxModEntry modEntry = inModifiedEbx[name]; + ebx[k++] = modEntry; + Modify(modEntry, j , true); + sha1[j++] = modEntry.Sha1; + if (strings.TryAdd(name, offset)) + { + offset += (uint)name.Length + 1; + } + } + + long resTypeOffset = inStream.Position + resCount * 2 * sizeof(uint); + long resMetaOffset = inStream.Position + resCount * 2 * sizeof(uint) + resCount * sizeof(uint); + long resRidOffset = inStream.Position + resCount * 2 * sizeof(uint) + resCount * sizeof(uint) + resCount * 0x10; + for (int i = 0; i < resCount; i++, j++) + { + uint nameOffset = inStream.ReadUInt32(endian); + uint originalSize = inStream.ReadUInt32(endian); + + long currentPos = inStream.Position; + inStream.Position = stringsOffset + nameOffset; + string name = inStream.ReadNullTerminatedString(); + + inStream.Position = resTypeOffset + i * sizeof(uint); + uint resType = inStream.ReadUInt32(); + + inStream.Position = resMetaOffset + i * 0x10; + byte[] resMeta = inStream.ReadBytes(0x10); + + inStream.Position = resRidOffset + i * sizeof(ulong); + ulong resRid = inStream.ReadUInt64(); + + if (inModInfo.Modified.Res.Contains(name)) + { + ResModEntry modEntry = inModifiedRes[name]; + res[i] = modEntry; + Modify(modEntry, j, false); + } + else + { + res[i] = new ResModEntry(name, sha1[j], originalSize, resRid, resType, resMeta); + } + + inStream.Position = currentPos; + + if (strings.TryAdd(name, offset)) + { + offset += (uint)name.Length + 1; + } + } + + k = resCount; + foreach (string name in inModInfo.Added.Res) + { + ResModEntry modEntry = inModifiedRes[name]; + res[k++] = modEntry; + Modify(modEntry, j , true); + sha1[j++] = modEntry.Sha1; + + if (strings.TryAdd(name, offset)) + { + offset += (uint)name.Length + 1; + } + } + + // TODO: how to handle the meta stuff + inStream.Position = metaOffset; + DbObjectDict chunkMeta = DbObject.Deserialize(inStream)!.AsDict(); + + inStream.Position = resRidOffset + resCount * sizeof(ulong); + for (int i = 0; i < chunkCount; i++, j++) + { + Guid id = inStream.ReadGuid(endian); + if (inModInfo.Modified.Chunks.Contains(id)) + { + var modEntry = inModifiedChunks[id]; + chunks[i] = modEntry; + Modify(modEntry, j, false); + } + else + { + chunks[i] = new ChunkModEntry(id, sha1[j], inStream.ReadUInt32(endian), inStream.ReadUInt32(endian)); + } + } + + k = chunkCount; + foreach (Guid id in inModInfo.Added.Chunks) + { + ChunkModEntry modEntry = inModifiedChunks[id]; + chunks[k++] = modEntry; + Modify(modEntry, j , true); + sha1[j++] = modEntry.Sha1; + + // TODO: add new chunk meta + } + + inStream.Position = startPos + size; + + // write new bundle + Block retVal; + using (DataStream stream = new(new MemoryStream())) + { + stream.WriteUInt32(0xDEADBEEF, Endian.Big); + + stream.WriteUInt32((uint)magic ^ Sdk.IO.BinaryBundle.GetSalt(), endian); + + stream.WriteInt32(sha1.Length, endian); + stream.WriteInt32(ebx.Length, endian); + stream.WriteInt32(res.Length, endian); + stream.WriteInt32(chunks.Length, endian); + stringsOffset = 32 + (containsSha1 ? sha1.Length * 20 : 0) + ebx.Length * 8 + res.Length * 36 + + chunks.Length * 24; // TODO: meta first? + stream.WriteUInt32((uint)stringsOffset, endian); + stream.WriteUInt32(0xDEADBEEF, endian); // TODO: metaOffset + stream.WriteUInt32(0xDEADBEEF, endian); // TODO: metaSize + + foreach (Sha1 value in sha1) + { + stream.WriteSha1(value, endian); + } + + foreach (EbxModEntry entry in ebx) + { + stream.WriteUInt32(strings[entry.Name], endian); + stream.WriteUInt32((uint)entry.OriginalSize, endian); + } + + resTypeOffset = stream.Position + res.Length * 2 * sizeof(uint); + resMetaOffset = stream.Position + res.Length * 2 * sizeof(uint) + res.Length * sizeof(uint); + resRidOffset = stream.Position + res.Length * 2 * sizeof(uint) + res.Length * sizeof(uint) + res.Length * 0x10; + for (int i = 0; i < res.Length; i++) + { + ResModEntry entry = res[i]; + stream.WriteUInt32(strings[entry.Name], endian); + stream.WriteUInt32((uint)entry.OriginalSize, endian); + + long currentPos = stream.Position; + stream.Position = resTypeOffset + i * sizeof(uint); + stream.WriteUInt32(entry.ResType); + + stream.Position = resMetaOffset + i * 0x10; + stream.Write(entry.ResMeta); + + stream.Position = resRidOffset + i * sizeof(ulong); + stream.WriteUInt64(entry.ResRid); + stream.Position = currentPos; + } + + foreach (ChunkModEntry entry in chunks) + { + stream.WriteGuid(entry.Id, endian); + stream.WriteUInt32(entry.LogicalOffset, endian); + stream.WriteUInt32(entry.LogicalSize, endian); + } + + // TODO: chunk meta + + Debug.Assert(stream.Position == stringsOffset + 4); + foreach (KeyValuePair pair in strings) + { + stream.Position = stringsOffset + 4 + pair.Value; + stream.WriteNullTerminatedString(pair.Key); + } + + stream.Position = 0; + retVal = new Block((int)stream.Length); + stream.ReadExactly(retVal); + } + + return retVal; + } +} \ No newline at end of file diff --git a/FrostyModSupport/FrostyModExecutor.Manifest2019.cs b/FrostyModSupport/FrostyModExecutor.Manifest2019.cs index 5444c4277..e9bd79f66 100644 --- a/FrostyModSupport/FrostyModExecutor.Manifest2019.cs +++ b/FrostyModSupport/FrostyModExecutor.Manifest2019.cs @@ -1,19 +1,342 @@ +using System.Diagnostics; +using Frosty.ModSupport.Archive; +using Frosty.ModSupport.ModEntries; using Frosty.ModSupport.ModInfos; +using Frosty.Sdk.DbObjectElements; +using Frosty.Sdk.Exceptions; +using Frosty.Sdk.IO; using Frosty.Sdk.Managers; +using Frosty.Sdk.Managers.Entries; using Frosty.Sdk.Managers.Infos; +using Frosty.Sdk.Managers.Loaders; +using Frosty.Sdk.Utils; namespace Frosty.ModSupport; public partial class FrostyModExecutor { - private void ModManifest2019(SuperBundleInstallChunk inSbIc, SuperBundleModInfo inModInfo) - { - string path = FileSystemManager.ResolvePath(inSbIc.Name); - if (string.IsNullOrEmpty(path)) + private void ModManifest2019(SuperBundleInstallChunk inSbIc, SuperBundleModInfo inModInfo, InstallChunkWriter inInstallChunkWriter) + { + string tocPath = Path.Combine(m_gamePatchPath, $"{inSbIc.Name}.toc"); + if (!File.Exists(tocPath)) { - return; + + } + + List< (string, uint, long)> bundles = new(); + List<(Guid, int, CasFileIdentifier, uint, uint)> chunks = new(); + + using (DataStream newBundleStream = new(new MemoryStream())) + { + using (BlockStream stream = BlockStream.FromFile(tocPath, true)) + { + stream.Position += sizeof(uint); // bundleHashMapOffset + uint bundleDataOffset = stream.ReadUInt32(Endian.Big); + int bundlesCount = stream.ReadInt32(Endian.Big); + + stream.Position += sizeof(uint); // chunkHashMapOffset + uint chunkGuidOffset = stream.ReadUInt32(Endian.Big); + int chunksCount = stream.ReadInt32(Endian.Big); + + // not used by any game rn, maybe crypto stuff + stream.Position += sizeof(uint); + stream.Position += sizeof(uint); + + uint namesOffset = stream.ReadUInt32(Endian.Big); + + uint chunkDataOffset = stream.ReadUInt32(Endian.Big); + int dataCount = stream.ReadInt32(Endian.Big); + + Manifest2019AssetLoader.Flags flags = (Manifest2019AssetLoader.Flags)stream.ReadInt32(Endian.Big); + + uint namesCount = 0; + uint tableCount = 0; + uint tableOffset = uint.MaxValue; + HuffmanDecoder? huffmanDecoder = null; + + if (flags.HasFlag(Manifest2019AssetLoader.Flags.HasCompressedNames)) + { + huffmanDecoder = new HuffmanDecoder(); + namesCount = stream.ReadUInt32(Endian.Big); + tableCount = stream.ReadUInt32(Endian.Big); + tableOffset = stream.ReadUInt32(Endian.Big); + } + + if (bundlesCount != 0) + { + if (flags.HasFlag(Manifest2019AssetLoader.Flags.HasCompressedNames)) + { + stream.Position = namesOffset; + huffmanDecoder!.ReadEncodedData(stream, namesCount, Endian.Big); + + stream.Position = tableOffset; + huffmanDecoder.ReadHuffmanTable(stream, tableCount, Endian.Big); + } + + stream.Position = bundleDataOffset; + BlockStream? sbStream = null; + for (int i = 0; i < bundlesCount; i++) + { + int nameOffset = stream.ReadInt32(Endian.Big); + uint bundleSize = stream.ReadUInt32(Endian.Big); + long bundleOffset = stream.ReadInt64(Endian.Big); + + // get name either from huffman table or raw string table at the end + string name; + if (flags.HasFlag(Manifest2019AssetLoader.Flags.HasCompressedNames)) + { + name = huffmanDecoder!.ReadHuffmanEncodedString(nameOffset); + } + else + { + long curPos = stream.Position; + stream.Position = namesOffset + nameOffset; + name = stream.ReadNullTerminatedString(); + stream.Position = curPos; + } + + int id = Utils.HashString(name + inSbIc.Name, true); + byte bundleLoadFlag = (byte)(bundleSize >> 30); + bundleSize &= 0x3FFFFFFFU; + + bundles.Add((name, (uint)newBundleStream.Position, bundleSize)); + + if (!inModInfo.Modified.Bundles.TryGetValue(id, out BundleModInfo? bundleModInfo)) + { + // load and write unmodified bundle + switch (bundleLoadFlag) + { + case 0: + sbStream ??= BlockStream.FromFile(tocPath.Replace(".toc", ".sb"), false); + sbStream.Position = bundleOffset; + sbStream.CopyTo(newBundleStream, (int)bundleSize); + break; + case 1: + long curPos = stream.Position; + stream.Position = bundleOffset; + stream.CopyTo(newBundleStream, (int)bundleSize); + stream.Position = curPos; + break; + default: + throw new UnknownValueException("bundle load flag", bundleLoadFlag); + } + } + else + { + // load, modify and write bundle + BlockStream bundleStream; + switch (bundleLoadFlag) + { + case 0: + sbStream ??= BlockStream.FromFile(tocPath.Replace(".toc", ".sb"), false); + bundleStream = sbStream; + break; + case 1: + bundleStream = stream; + break; + default: + throw new UnknownValueException("bundle load flag", bundleLoadFlag); + } + + (Block BundleMeta, List<(CasFileIdentifier, uint, uint)> Files, bool IsInline) bundle = LoadBundle(bundleStream, bundleOffset, bundleModInfo, inInstallChunkWriter); + + // TODO: write modified bundle + using (DataStream bundleWriter = new(new MemoryStream())) + { + bundleWriter.WriteInt32(bundle.IsInline ? 0x20 : 0, Endian.Big); + bundleWriter.WriteInt32(bundle.IsInline ? bundle.BundleMeta.Size : 0, Endian.Big); + bundleWriter.WriteUInt32(0xDEADBEEF, Endian.Big); // fileIdentifiedFlags offset + bundleWriter.WriteInt32(bundle.Files.Count, Endian.Big); + bundleWriter.WriteUInt32(0xDEADBEEF, Endian.Big); // fileIdentifier offset + bundleWriter.WriteUInt32(0xDEADBEEF, Endian.Big); // unused + bundleWriter.WriteUInt32(0xDEADBEEF, Endian.Big); // unused + bundleWriter.WriteUInt32(0, Endian.Big); // unused + + if (bundle.IsInline) + { + bundleWriter.Write(bundle.BundleMeta); + } + else + { + bundle.Files[0] = inInstallChunkWriter.WriteData(Utils.GenerateSha1(bundle.BundleMeta), bundle.BundleMeta); + } + + bundleWriter.Pad(4); + + byte[] fileFlags = new byte[bundle.Files.Count]; + long fileIdentifierOffset = bundleWriter.Position; + CasFileIdentifier current = default; + for (int j = 0; j < bundle.Files.Count; j++) + { + (CasFileIdentifier, uint, uint) file = bundle.Files[j]; + if (file.Item1 == current && j != 0) + { + fileFlags[j] = 0; + } + else + { + if (file.Item1.InstallChunkIndex > byte.MaxValue) + { + fileFlags[j] = 0x80; + bundleWriter.WriteUInt64(CasFileIdentifier.ToFileIdentifierLong(file.Item1), Endian.Big); + } + else + { + fileFlags[j] = 1; + bundleWriter.WriteUInt32(CasFileIdentifier.ToFileIdentifier(file.Item1), Endian.Big); + } + + + } + bundleWriter.WriteUInt32(file.Item2, Endian.Big); + bundleWriter.WriteUInt32(file.Item3, Endian.Big); + } + + long fileFlagOffset = bundleWriter.Position; + foreach (byte flag in fileFlags) + { + bundleWriter.WriteByte(flag); + } + + bundleWriter.Position = 8; + bundleWriter.WriteUInt32((uint)fileFlagOffset); + bundleWriter.Position = 16; + bundleWriter.WriteUInt32((uint)fileIdentifierOffset); + } + + bundle.BundleMeta.Dispose(); + + // remove bundle so we can check if the base superbundle needs to be loaded to modify a base bundle + inModInfo.Modified.Bundles.Remove(id); + } + } + huffmanDecoder?.Dispose(); + } + } + } + + bool encodeStrings = true; + uint offset = encodeStrings ? 0x3Cu : 0x30u; + using (DataStream stream = new(new MemoryStream())) + { + stream.WriteUInt32(offset, Endian.Big); + offset += (uint)bundles.Count * sizeof(int); + stream.WriteUInt32(offset, Endian.Big); + offset += (uint)bundles.Count * (sizeof(int) + sizeof(uint) + sizeof(long)); + stream.WriteInt32(bundles.Count, Endian.Big); + + stream.WriteUInt32(offset, Endian.Big); + offset += (uint)chunks.Count * sizeof(int); + stream.WriteUInt32(offset, Endian.Big); + offset += (uint)chunks.Count * (16 + sizeof(int)); + stream.WriteInt32(chunks.Count, Endian.Big); + } + } + + private (Block, List<(CasFileIdentifier, uint, uint)>, bool) LoadBundle(BlockStream stream, long inOffset, BundleModInfo inModInfo, InstallChunkWriter inInstallChunkWriter) + { + long curPos = stream.Position; + + stream.Position = inOffset; + + int bundleOffset = stream.ReadInt32(Endian.Big); + int bundleSize = stream.ReadInt32(Endian.Big); + uint locationOffset = stream.ReadUInt32(Endian.Big); + int totalCount = stream.ReadInt32(Endian.Big); + uint dataOffset = stream.ReadUInt32(Endian.Big); + + // not used by any game rn, again maybe crypto stuff + stream.Position += sizeof(uint); + stream.Position += sizeof(uint); + // maybe count for the offsets above + stream.Position += sizeof(int); + + bool inlineBundle = !(bundleOffset == 0 && bundleSize == 0); + + stream.Position = inOffset + locationOffset; + + Block fileIdentifierFlags = new(totalCount); + stream.ReadExactly(fileIdentifierFlags); + + CasFileIdentifier file = default; + int currentIndex = 0; + + List<(CasFileIdentifier, uint, uint)> files = new(totalCount); + for (; currentIndex < totalCount; currentIndex++) + { + file = ReadCasFileIdentifier(stream, fileIdentifierFlags[currentIndex], file); + + files.Add((file, stream.ReadUInt32(Endian.Big), stream.ReadUInt32(Endian.Big))); + } + + Block bundleMeta; + if (inlineBundle) + { + stream.Position = inOffset + bundleOffset; + bundleMeta = BinaryBundle.Modify(stream, inModInfo, m_modifiedEbx, m_modifiedRes, m_modifiedChunks, + (entry, i, isAdded) => + { + if (isAdded) + { + files.Insert(i, inInstallChunkWriter.GetFileInfo(entry.Sha1)); + } + else + { + files[i] = inInstallChunkWriter.GetFileInfo(entry.Sha1); + } + }); + + // go to the start of the data + stream.Position = inOffset + dataOffset; + } + else + { + stream.Position = inOffset + dataOffset; + file = ReadCasFileIdentifier(stream, fileIdentifierFlags[0], file); + uint offset = stream.ReadUInt32(Endian.Big); + int size = stream.ReadInt32(Endian.Big); + string path = FileSystemManager.GetFilePath(file); + if (string.IsNullOrEmpty(path)) + { + throw new Exception("Corrupted data. File for bundle does not exist."); + } + using (BlockStream bundleStream = BlockStream.FromFile(path, offset, size)) + { + bundleMeta = BinaryBundle.Modify(stream, inModInfo, m_modifiedEbx, m_modifiedRes, m_modifiedChunks, + (entry, i, isAdded) => + { + if (isAdded) + { + files.Insert(i + 1, inInstallChunkWriter.GetFileInfo(entry.Sha1)); + } + else + { + files[i + 1] = inInstallChunkWriter.GetFileInfo(entry.Sha1); + } + }); + Debug.Assert(bundleStream.Position == bundleStream.Length, "We did not read the bundle meta completely"); + } } + fileIdentifierFlags.Dispose(); + stream.Position = curPos; + + return (bundleMeta, files, inlineBundle); + } + + private CasFileIdentifier ReadCasFileIdentifier(DataStream stream, byte inFlag, CasFileIdentifier current) + { + switch (inFlag) + { + case 0: + return current; + case 1: + return CasFileIdentifier.FromFileIdentifier(stream.ReadUInt32(Endian.Big)); + case 0x80: + return CasFileIdentifier.FromFileIdentifier(stream.ReadUInt32(Endian.Big), stream.ReadUInt32(Endian.Big)); + default: + throw new UnknownValueException("file identifier flag", inFlag); + } } } \ No newline at end of file diff --git a/FrostyModSupport/FrostyModExecutor.cs b/FrostyModSupport/FrostyModExecutor.cs index 46c6d67ad..09e33744d 100644 --- a/FrostyModSupport/FrostyModExecutor.cs +++ b/FrostyModSupport/FrostyModExecutor.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Reflection; using System.Text.Json; +using Frosty.ModSupport.Archive; using Frosty.ModSupport.Attributes; using Frosty.ModSupport.Interfaces; using Frosty.ModSupport.Mod; @@ -30,6 +31,12 @@ public partial class FrostyModExecutor private readonly Dictionary m_bundleToSuperBundleMapping = new(); private readonly Dictionary m_handlers = new(); + + private readonly Dictionary m_installChunkWriters = new(); + + private string m_patchPath; + private string m_modDataPath; + private string m_gamePatchPath; /// /// Generates a directory containing the modded games data. @@ -39,14 +46,14 @@ public partial class FrostyModExecutor public Errors GenerateMods(string modPackName, params string[] modPaths) { // define some paths we are going to need - string patchPath = FileSystemManager.Sources.Count == 1 + m_patchPath = FileSystemManager.Sources.Count == 1 ? FileSystemSource.Base.Path : FileSystemSource.Patch.Path; - string modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName, patchPath); - string gamePatchPath = Path.Combine(FileSystemManager.BasePath, patchPath); + m_modDataPath = Path.Combine(FileSystemManager.BasePath, "ModData", modPackName, m_patchPath); + m_gamePatchPath = Path.Combine(FileSystemManager.BasePath, m_patchPath); // check if we need to generate new data - string modInfosPath = Path.Combine(modDataPath, "mods.json"); + string modInfosPath = Path.Combine(m_modDataPath, "mods.json"); List modInfos = GenerateModInfoList(modPaths); if (File.Exists(modInfosPath)) { @@ -109,15 +116,17 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) } // clear old generated mod data - Directory.Delete(modDataPath, true); - Directory.CreateDirectory(modDataPath); + Directory.Delete(m_modDataPath, true); + Directory.CreateDirectory(m_modDataPath); // modify the superbundles and write them to mod data foreach (KeyValuePair sb in m_superBundleModInfos) { SuperBundleInstallChunk sbic = FileSystemManager.GetSuperBundleInstallChunk(sb.Key); + + // write all data in this superbundle to cas files in the correct install chunk + InstallChunkWriter installChunkWriter = WriteCasArchives(sb.Value, sbic); - // TODO: we need to write the data to cas files (should we write them per bundle or per superbundle) and store the references, so we can write them into the cat files/directly in the bundle switch (FileSystemManager.BundleFormat) { case BundleFormat.Dynamic2018: @@ -132,9 +141,9 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) } // create symbolic links for everything that is in gamePatchPath but not in modDataPath - foreach (string file in Directory.EnumerateFiles(gamePatchPath, string.Empty, SearchOption.AllDirectories)) + foreach (string file in Directory.EnumerateFiles(m_gamePatchPath, string.Empty, SearchOption.AllDirectories)) { - string modPath = Path.Combine(modDataPath, Path.GetRelativePath(gamePatchPath, file)); + string modPath = Path.Combine(m_modDataPath, Path.GetRelativePath(m_gamePatchPath, file)); if (!File.Exists(modPath)) { // TODO: check if we need to create the directory first @@ -145,6 +154,26 @@ public Errors GenerateMods(string modPackName, params string[] modPaths) return Errors.Success; } + private InstallChunkWriter WriteCasArchives(SuperBundleModInfo inModInfo, SuperBundleInstallChunk sbIc) + { + if (!m_installChunkWriters.TryGetValue(sbIc.InstallChunk.Id, out InstallChunkWriter? installChunkWriter)) + { + m_installChunkWriters.Add(sbIc.InstallChunk.Id, installChunkWriter = new InstallChunkWriter(sbIc.InstallChunk, m_gamePatchPath, m_modDataPath)); + } + + foreach (Sha1 sha1 in inModInfo.Data) + { + (Block, bool) data = GetData(sha1); + installChunkWriter.WriteData(sha1, data.Item1); + if (data.Item2) + { + data.Item1.Dispose(); + } + } + + return installChunkWriter; + } + private void LoadHandlers() { foreach (string handler in Directory.EnumerateFiles("Handlers")) @@ -403,6 +432,7 @@ private void ProcessModResources(IResourceContainer container) foreach (int superBundle in entry.SuperBundleInstallChunks) { SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); + sb.Data.Add(resource.Sha1); sb.Modified.Chunks.Add(id); } } @@ -411,6 +441,7 @@ private void ProcessModResources(IResourceContainer container) foreach (int superBundle in chunk.AddedSuperBundles) { SuperBundleModInfo sb = GetSuperBundleModInfo(superBundle); + sb.Data.Add(resource.Sha1); sb.Added.Chunks.Add(id); } @@ -435,6 +466,11 @@ private void ProcessModResources(IResourceContainer container) { SuperBundleModInfo sb = GetSuperBundleModInfoFromBundle(addedBundle); + if (resource.Sha1 != Sha1.Zero) + { + sb.Data.Add(resource.Sha1); + } + if (!sb.Modified.Bundles.TryGetValue(addedBundle, out BundleModInfo? modInfo)) { modInfo = new BundleModInfo(); @@ -483,6 +519,11 @@ private void ProcessModResources(IResourceContainer container) { SuperBundleModInfo sb = GetSuperBundleModInfoFromBundle(modifiedBundle); + if (resource.Sha1 != Sha1.Zero) + { + sb.Data.Add(resource.Sha1); + } + if (!sb.Modified.Bundles.TryGetValue(modifiedBundle, out BundleModInfo? modInfo)) { modInfo = new BundleModInfo(); @@ -577,16 +618,16 @@ private static List GenerateModInfoList(IEnumerable modPaths) return modInfoList; } - private Block GetData(Sha1 sha1) + private (Block, bool) GetData(Sha1 sha1) { if (m_data.TryGetValue(sha1, out ResourceData? data)) { - return data.GetData(); + return (data.GetData(), true); } if (m_data2.TryGetValue(sha1, out Block? block)) { - return block; + return (block, false); } throw new Exception(); diff --git a/FrostyModSupport/ModEntries/ChunkModEntry.cs b/FrostyModSupport/ModEntries/ChunkModEntry.cs index 9147a5919..2ded7797b 100644 --- a/FrostyModSupport/ModEntries/ChunkModEntry.cs +++ b/FrostyModSupport/ModEntries/ChunkModEntry.cs @@ -30,4 +30,12 @@ public ChunkModEntry(ChunkModResource inResource, long inSize) Size = inSize; } + + public ChunkModEntry(Guid inId, Sha1 inSha1, uint inLogicalOffset, uint inLogicalSize) + { + Id = inId; + Sha1 = inSha1; + LogicalOffset = inLogicalOffset; + LogicalSize = inLogicalSize; + } } \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/EbxModEntry.cs b/FrostyModSupport/ModEntries/EbxModEntry.cs index 51ff1e6ad..46db0da2e 100644 --- a/FrostyModSupport/ModEntries/EbxModEntry.cs +++ b/FrostyModSupport/ModEntries/EbxModEntry.cs @@ -19,4 +19,11 @@ public EbxModEntry(EbxModResource inResource, long inSize) OriginalSize = inResource.OriginalSize; Size = inSize; } + + public EbxModEntry(string inName, Sha1 inSha1, long inOriginalSize) + { + Name = inName; + Sha1 = inSha1; + OriginalSize = inOriginalSize; + } } \ No newline at end of file diff --git a/FrostyModSupport/ModEntries/ResModEntry.cs b/FrostyModSupport/ModEntries/ResModEntry.cs index c82cf73d5..08faa88ac 100644 --- a/FrostyModSupport/ModEntries/ResModEntry.cs +++ b/FrostyModSupport/ModEntries/ResModEntry.cs @@ -25,4 +25,15 @@ public ResModEntry(ResModResource inResource, long inSize) ResMeta = inResource.ResMeta; Size = inSize; } + + public ResModEntry(string inName, Sha1 inSha1, long inOriginalSize, ulong inResRid, uint inResType, + byte[] inResMeta) + { + Name = inName; + Sha1 = inSha1; + OriginalSize = inOriginalSize; + ResRid = inResRid; + ResType = inResType; + ResMeta = inResMeta; + } } \ No newline at end of file diff --git a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs index a67bb326c..48ed5d1d7 100644 --- a/FrostyModSupport/ModInfos/SuperBundleModInfo.cs +++ b/FrostyModSupport/ModInfos/SuperBundleModInfo.cs @@ -1,8 +1,11 @@ -namespace Frosty.ModSupport.ModInfos; +using Frosty.Sdk; + +namespace Frosty.ModSupport.ModInfos; public class SuperBundleModInfo { public SuperBundleModAction Added = new(); public SuperBundleModAction Removed = new(); public SuperBundleModAction Modified = new(); + public HashSet Data = new(); } \ No newline at end of file diff --git a/FrostySdk/IO/BinaryBundle.cs b/FrostySdk/IO/BinaryBundle.cs index 12662374f..9c63df1dd 100644 --- a/FrostySdk/IO/BinaryBundle.cs +++ b/FrostySdk/IO/BinaryBundle.cs @@ -11,7 +11,7 @@ namespace Frosty.Sdk.IO; public class BinaryBundle { - private enum Magic : uint + public enum Magic : uint { Standard = 0xED1CEDB8, Kelvin = 0xC3889333, @@ -141,7 +141,7 @@ private BinaryBundle(DataStream stream) /// is the only game that uses "arie". /// /// The salt, that the current game uses. - private static uint GetSalt() + public static uint GetSalt() { const uint pecm = 0x7065636D; const uint pecn = 0x7065636E; @@ -164,7 +164,7 @@ private static uint GetSalt() /// Only the games using the format have their own magic, the rest uses . /// /// The magic the current game uses. - private static Magic GetMagic() + public static Magic GetMagic() { switch (FileSystemManager.BundleFormat) { diff --git a/FrostySdk/Managers/Infos/CasFileIdentifier.cs b/FrostySdk/Managers/Infos/CasFileIdentifier.cs index 0a610d305..99aec6596 100644 --- a/FrostySdk/Managers/Infos/CasFileIdentifier.cs +++ b/FrostySdk/Managers/Infos/CasFileIdentifier.cs @@ -22,6 +22,11 @@ public static uint ToFileIdentifier(CasFileIdentifier file) { return (uint)((file.IsPatch ? 1 << 16 : 0) | (file.InstallChunkIndex << 8) | (file.CasIndex)); } + + public static ulong ToFileIdentifierLong(CasFileIdentifier file) + { + return (ulong)((file.IsPatch ? 1L << 48 : 0) | ((long)file.InstallChunkIndex << 16) | (file.CasIndex)); + } public static CasFileIdentifier FromManifestFileIdentifier(uint file) { diff --git a/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs b/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs index 7a5a3471c..6af9673dd 100644 --- a/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs +++ b/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs @@ -22,7 +22,7 @@ public enum Code public class Manifest2019AssetLoader : IAssetLoader { [Flags] - private enum Flags + public enum Flags { HasBaseBundles = 1 << 0, // base toc has bundles that the patch doesnt have HasBaseChunks = 1 << 1, // base toc has chunks that the patch doesnt have @@ -263,7 +263,7 @@ private void LoadBundle(DataStream stream, long inOffset, uint inSize, ref Bundl stream.Position = inOffset + locationOffset; - Block fileIdentifierFlags = new Block(totalCount); + Block fileIdentifierFlags = new(totalCount); stream.ReadExactly(fileIdentifierFlags); // the flags should be the last thing in the bundle