Merge branch 'improve-fastbuild-unity-generation' into 'main'

Improve unity file generation to avoid fastbuild cache misses

See merge request Sharpmake/sharpmake!489

Sharpmake.Generators/FastBuild/Bff.Util.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Ubisoft. All Rights Reserved.
+// Copyright (c) Ubisoft. All Rights Reserved.
 // Licensed under the Apache 2.0 License. See LICENSE.md in the project root for license information.
 
 using System;
@@ -31,7 +31,7 @@ public class Unity
             public string UnityNumFiles = FileGeneratorUtilities.RemoveLineTag; // (optional) Number of Unity files to generate (default 1)
             public string UnityPCH = FileGeneratorUtilities.RemoveLineTag; // (optional) Precompiled Header file to add to generated Unity files
             public string UseRelativePaths = FileGeneratorUtilities.RemoveLineTag; // (optional) Use relative paths for generated Unity files
-
+            public Byte UnitySectionBucket = 0; // Internal sharpmake field used to force separate unity sections in certain cases.
             public const string DefaultUnityInputPatternExtension = ".cpp";
             public const string DefaultUnityOutputPatternExtension = "Unity*.cpp";
 
@@ -59,6 +59,7 @@ public override int GetHashCode()
                     hash = hash * 23 + UnityNumFiles.GetDeterministicHashCode();
                     hash = hash * 23 + UnityPCH.GetDeterministicHashCode();
                     hash = hash * 23 + UseRelativePaths.GetDeterministicHashCode();
+                    hash = hash * 23 + UnitySectionBucket;
 
                     return hash;
                 }
@@ -94,7 +95,8 @@ private bool Equals(Unity unity)
                     && string.Equals(UnityOutputPattern, unity.UnityOutputPattern)
                     && string.Equals(UnityNumFiles, unity.UnityNumFiles)
                     && string.Equals(UnityPCH, unity.UnityPCH)
-                    && string.Equals(UseRelativePaths, unity.UseRelativePaths);
+                    && string.Equals(UseRelativePaths, unity.UseRelativePaths)
+                    && UnitySectionBucket == unity.UnitySectionBucket;
             }
         }
 

Sharpmake.Generators/FastBuild/Bff.cs

@@ -2000,6 +2000,7 @@ private void ConfigureUnities(IGenerationContext context, Dictionary<Project.Con
                 UnityInputExcludedFiles = fastBuildUnityInputExcludedfiles,
                 UnityInputPattern = fastBuildUnityInputPattern,
                 UseRelativePaths = conf.FastBuildUnityUseRelativePaths ? "true" : FileGeneratorUtilities.RemoveLineTag,
+                UnitySectionBucket = conf.FastBuildUnitySectionBucket,
             };
 
             // _unities being a dictionary, a new entry will be created only

Sharpmake.UnitTests/BffUnityResolverTest.cs

@@ -57,12 +57,12 @@ public void HashUnityResolver()
             var unityLogNames = GetProjectUnityLogNames();
 
             var expectedLogNames = new List<string> {
-                    "SimpleProject_vs2017_Debug : SimpleProject_unity_B213A137",   // Debug and Release have the same unity config, so hash is identical.
-                    "SimpleProject_vs2017_Release : SimpleProject_unity_B213A137",
-                    "SimpleProject_vs2017_Retail : SimpleProject_unity_F850C659",
-                    "SimpleProject_vs2019_Debug : SimpleProject_unity_A6BA8495",   // vs2019 values follow same pattern (debug == release),
-                    "SimpleProject_vs2019_Release : SimpleProject_unity_A6BA8495", // but different hash values because of different conf.FastBuildUnityUseRelativePaths from vs2017.
-                    "SimpleProject_vs2019_Retail : SimpleProject_unity_80CD78B1",
+                    "SimpleProject_vs2017_Debug : SimpleProject_unity_82533B9B",   // Debug and Release have the same unity config, so hash is identical.
+                    "SimpleProject_vs2017_Release : SimpleProject_unity_82533B9B",
+                    "SimpleProject_vs2017_Retail : SimpleProject_unity_0BAA6A15",
+                    "SimpleProject_vs2019_Debug : SimpleProject_unity_67D9F899",   // vs2019 values follow same pattern (debug == release),
+                    "SimpleProject_vs2019_Release : SimpleProject_unity_67D9F899", // but different hash values because of different conf.FastBuildUnityUseRelativePaths from vs2017.
+                    "SimpleProject_vs2019_Retail : SimpleProject_unity_3DFB75FD",
                 };
             CollectionAssert.AreEqual(expectedLogNames, unityLogNames);
         }

Sharpmake/Project.Configuration.cs

@@ -1596,6 +1596,39 @@ public string FastBuildUnityPath
             /// </summary>
             public bool FastBuildUnityUseRelativePaths = false;
 
+            /// <summary>
+            /// Give a same value to configurations with same fastbuild unity settings that you want to keep together. If you want to have developer
+            /// machines have fastbuild cache hits but the section settings are not exactly the same for some targets you will need this field to
+            /// to have the same unity sections than on your build machines(typically at Ubisoft, only have the build machines have the cache in read-write mode).
+            /// </summary>
+            /// <remarks>
+            /// With this field we can force some extra fastbuild unity sections even for sections with identical settings.
+            /// Sharpmake will take into account the bucket number when creating its internal unity objects and this will let us create artifical delimitation 
+            /// of setting sections.
+            /// This field should be used only if FragmentHashUnityResolver is used.
+            /// </remarks>
+            /// <example>
+            /// You have two targets: Debug and Release. You want to enable deoptimization on debug target on developer machine but never on build machines
+            /// In a Configure method you do this:
+            /// if (IsBuildMachine() || target.Optimization > Optimization.Debug )
+            /// {
+            ///     conf.FastBuildUnityInputIsolateWritableFiles = false;
+            ///     conf.FastBuildDeoptimization = Configuration.DeoptimizationWritableFiles.NoDeoptimization;
+            ///}
+            ///else
+            ///{
+            ///    conf.FastBuildUnityInputIsolateWritableFilesLimit = 50;
+            ///    conf.FastBuildDeoptimization = true;
+            ///}
+            /// Without this change and the code below we have different unity sections on build machine and developper machines, causing fastbuild cache misses.
+            /// We now do this in a Configure method:
+            /// conf.FastBuildUnitySectionBucket = (byte)(target.Optimization > Optimization.Debug ? 1 : 0);
+            /// By doing this we force two separate unity sections on build machines(same as on developer machine).
+            /// 
+            /// Important: This assumes that you configured Sharpmake to use FragmentHashUnityResolver as the unity resolver.
+            /// </example>
+            public Byte FastBuildUnitySectionBucket = 0;
+
             /// <summary>
             /// Gets or sets whether to generate a FASTBuild (.bff) file when using FASTBuild.
             /// </summary>