From 395d4387ba76ee8113f85dcde79c639644391138 Mon Sep 17 00:00:00 2001
From: Colin Cornaby <colin.cornaby@mac.com>
Date: Wed, 4 Dec 2024 00:15:09 -0800
Subject: [PATCH] Fixing normal based UV calculation in Metal shader

The matrix multiply order was incorrect. Also added some clarity to the varaible names.
---
 .../ShaderSrc/FixedPipelineShaders.metal          | 15 ++++++++-------
 .../pfMetalPipeline/ShaderSrc/ShaderTypes.h       |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/FixedPipelineShaders.metal b/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/FixedPipelineShaders.metal
index 7623c5c186..d8a8496fed 100644
--- a/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/FixedPipelineShaders.metal
+++ b/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/FixedPipelineShaders.metal
@@ -246,10 +246,10 @@ vertex ColorInOut pipelineVertexShader(Vertex in [[stage_in]],
     // Fog
     out.fogColor = uniforms.calcFog(vCamPosition);
 
-    const float4 normal = (uniforms.localToWorldMatrix * float4(in.normal, 0.f)) * uniforms.worldToCameraMatrix;
+    const float4 cameraSpaceNormal = normalize(((float4(in.normal, 0.f) * uniforms.localToWorldMatrix) * uniforms.worldToCameraMatrix));
 
     for (size_t layer=0; layer<num_layers; layer++) {
-        (&out.texCoord1)[layer] = uniforms.sampleLocation(layer, &in.texCoord1, normal, vCamPosition);
+        (&out.texCoord1)[layer] = uniforms.sampleLocation(layer, &in.texCoord1, cameraSpaceNormal, vCamPosition);
     }
 
     out.position = vCamPosition * uniforms.projectionMatrix;
@@ -260,7 +260,7 @@ vertex ColorInOut pipelineVertexShader(Vertex in [[stage_in]],
 constexpr void blendFirst(half4 srcSample, thread half4 &destSample, const uint32_t blendFlags);
 constexpr void blend(half4 srcSample, thread half4 &destSample, uint32_t blendFlags);
 
-float3 VertexUniforms::sampleLocation(size_t index, thread float3 *texCoords, const float4 normal, const float4 camPosition) constant
+float3 VertexUniforms::sampleLocation(size_t index, thread float3 *texCoords, const float4 cameraSpaceNormal, const float4 camPosition) constant
 {
     const uint32_t UVWSrc = uvTransforms[index].UVWSrc;
     float4x4 matrix = uvTransforms[index].transform;
@@ -348,7 +348,7 @@ float3 VertexUniforms::sampleLocation(size_t index, thread float3 *texCoords, co
     switch (UVWSrc) {
     case kUVWNormal:
         {
-            sampleCoord = normal * matrix;
+            sampleCoord = cameraSpaceNormal * matrix;
         }
         break;
     case kUVWPosition:
@@ -358,7 +358,7 @@ float3 VertexUniforms::sampleLocation(size_t index, thread float3 *texCoords, co
         break;
     case kUVWReflect:
         {
-            sampleCoord = reflect(normalize(camPosition), normalize(normal)) * matrix;
+            sampleCoord = reflect(normalize(camPosition), cameraSpaceNormal) * matrix;
         }
         break;
     default:
@@ -643,10 +643,11 @@ vertex ColorInOut shadowCastVertexShader(Vertex in
     // Fog
     out.fogColor = uniforms.calcFog(vCamPosition);
 
-    const float4 normal = (uniforms.localToWorldMatrix * float4(in.normal, 0.f)) * uniforms.worldToCameraMatrix;
+    // FIXME: Shadow casting doesn't use normals. Simplify texture sampling.
+    const float4 cameraSpaceNormal = normalize(((float4(in.normal, 0.f) * uniforms.localToWorldMatrix) * uniforms.worldToCameraMatrix));
 
     for (size_t layer=0; layer<num_layers; layer++) {
-        (&out.texCoord1)[layer] = uniforms.sampleLocation(layer, &in.texCoord1, normal, vCamPosition);
+        (&out.texCoord1)[layer] = uniforms.sampleLocation(layer, &in.texCoord1, cameraSpaceNormal, vCamPosition);
     }
 
     out.position = vCamPosition * uniforms.projectionMatrix;
diff --git a/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/ShaderTypes.h b/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/ShaderTypes.h
index 2845921eb8..1b736d7547 100644
--- a/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/ShaderTypes.h
+++ b/Sources/Plasma/FeatureLib/pfMetalPipeline/ShaderSrc/ShaderTypes.h
@@ -194,7 +194,7 @@ struct VertexUniforms
 
     UVOutDescriptor uvTransforms[8];
 #ifdef __METAL_VERSION__
-    float3 sampleLocation(size_t index, thread float3 *texCoords, const float4 normal, const float4 camPosition) constant;
+    float3 sampleLocation(size_t index, thread float3 *texCoords, const float4 cameraSpaceNormal, const float4 camPosition) constant;
     half4 calcFog(float4 camPosition) constant;
 #endif
 };