feat: Materials can now use the screen texture (#2849)

https://github.com/excaliburjs/Excalibur/assets/612071/ade92ec3-caff-4835-8410-74f4ebbbe421

This PR adds a new feature for materials that allows them to reference the screen texture. They way it works is it's the screen's framebuffer RIGHT before the material draw call (if you include the complete drawing you get the infinite mirror artifact which is pretty unusable)

2 new uniforms

* `u_screen_texture` - This is the texture of the screen right before the material draw call
* `u_time_ms` - This is the milliseconds since page navigation (`performance.now()` under the hood)

2 new attribute/varyings

* `a_screenuv` - The vertex attribute corresponding to the screen uv relative to the current graphic
* `v_screenuv` - The fragment varying corresponding to the screen uv relative to the current graphic

Finally there is a new convenience api for updating shader values in materials. `.update(shader => {...})`

```typescript
  game.input.pointers.primary.on('move', evt => {
    heartActor.pos = evt.worldPos;
    swirlMaterial.update(shader => {
      shader.trySetUniformFloatVector('iMouse', evt.worldPos);
    });
  });
```

.vscode/settings.json

@@ -24,6 +24,6 @@
   },
   "typescript.tsdk": "./node_modules/typescript/lib",
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   }
 }

CHANGELOG.md

@@ -15,7 +15,15 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
-- 
+- Materials have a new convenience method for updating uniforms
+  ```typescript
+  game.input.pointers.primary.on('move', evt => {
+    heartActor.pos = evt.worldPos;
+    swirlMaterial.update(shader => {
+      shader.trySetUniformFloatVector('iMouse', evt.worldPos);
+    });
+  });
+  ```
 
 
 ### Fixed
@@ -24,7 +32,13 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Updates
 
-- 
+- Materials can now reference a new uniform for the screen texture and a screen uv attribute in their fragment shaders
+  * `u_screen_texture` - This is the texture of the screen right before the material draw call
+  * `a_screenuv` - The vertex attribute corresponding to the screen uv relative to the current graphic
+  * `v_screenuv` - The fragment varying corresponding to the screen uv relative to the current graphic
+
+- Materials can now reference the current time in their shaders
+  * `u_time_ms` - This is the ms since page navigation (performance.now() under the hood)
 
 ### Changed
 

sandbox/tests/material/index.ts

@@ -1,7 +1,7 @@
 /// <reference path="../../lib/excalibur.d.ts" />
 
 // identity tagged template literal lights up glsl-literal vscode plugin
-var glsl = x => x;
+var glsl = x => x[0];
 
 var game = new ex.Engine({
   canvasElementId: 'game',
@@ -21,42 +21,41 @@ var loader = new ex.Loader([tex, heartImage, background]);
 var outline = glsl`#version 300 es
 precision mediump float;
 
-uniform float iTime;
-
+uniform float u_time_ms;
 uniform sampler2D u_graphic;
 
 in vec2 v_uv;
-
+in vec2 v_screenuv;
 out vec4 fragColor;
 
 vec3 hsv2rgb(vec3 c){
-	vec4 K=vec4(1.,2./3.,1./3.,3.);
-	return c.z*mix(K.xxx,clamp(abs(fract(c.x+K.xyz)*6.-K.w)-K.x, 0., 1.),c.y);
+  vec4 K=vec4(1.,2./3.,1./3.,3.);
+  return c.z*mix(K.xxx,clamp(abs(fract(c.x+K.xyz)*6.-K.w)-K.x, 0., 1.),c.y);
 }
 
 void main() {
   const float TAU = 6.28318530;
-	const float steps = 4.0; // up/down/left/right pixels
-
-	float radius = 2.0;
-  vec3 outlineColorHSL = vec3(sin(iTime/2.0) * 1., 1., 1.);
+  const float steps = 4.0; // up/down/left/right pixels
+  float radius = 2.0;
+  float time_sec = u_time_ms / 1000.;
 
+  vec3 outlineColorHSL = vec3(sin(time_sec/2.0) * 1., 1., 1.);
   vec2 aspect = 1.0 / vec2(textureSize(u_graphic, 0));
 
-	for (float i = 0.0; i < TAU; i += TAU / steps) {
-		// Sample image in a circular pattern
+  for (float i = 0.0; i < TAU; i += TAU / steps) {
+    // Sample image in a circular pattern
     vec2 offset = vec2(sin(i), cos(i)) * aspect * radius;
-		vec4 col = texture(u_graphic, v_uv + offset);
-		
-		// Mix outline with background
-		float alpha = smoothstep(0.5, 0.7, col.a);
-		fragColor = mix(fragColor, vec4(hsv2rgb(outlineColorHSL), 1.0), alpha); // apply outline
-	}
-	
+    vec4 col = texture(u_graphic, v_uv + offset);
+
+    // Mix outline with background
+    float alpha = smoothstep(0.5, 0.7, col.a);
+    fragColor = mix(fragColor, vec4(hsv2rgb(outlineColorHSL), 1.0), alpha); // apply outline
+  }
+
   // Overlay original texture
-	vec4 mat = texture(u_graphic, v_uv);
-	float factor = smoothstep(0.5, 0.7, mat.a);
-	fragColor = mix(fragColor, mat, factor);
+  vec4 mat = texture(u_graphic, v_uv);
+  float factor = smoothstep(0.5, 0.7, mat.a);
+  fragColor = mix(fragColor, mat, factor);
 }
 `
 
@@ -70,7 +69,7 @@ uniform sampler2D u_graphic;
 
 uniform vec2 u_resolution;
 
-uniform float iTime;
+uniform float u_time_ms;
 
 uniform vec2 iMouse;
 
@@ -83,21 +82,19 @@ uniform float u_opacity;
 out vec4 fragColor;
 
 void main() {
-   vec4 color = u_color;
-
+  vec4 color = u_color;
+  float time_sec = u_time_ms / 1000.;
   float effectRadius = .5;
-  float effectAngle = mod(iTime/2., 2.)  * 3.14159;
+  float effectAngle = mod(time_sec/2., 2.)  * 3.14159;
 
   vec2 size = u_size.xy;
-
   vec2 center = iMouse.xy / u_size.xy;
   vec2 uv = v_uv.xy - center;
 
   float len = length(uv * vec2(size.x / size.y, 1.));
   float angle = atan(uv.y, uv.x) + effectAngle * smoothstep(effectRadius, 0., len);
   float radius = length(uv);
   vec2 newUv = vec2(radius * cos(angle), radius * sin(angle)) + center;
-
    color = texture(u_graphic, newUv);
    color.rgb = color.rgb * u_opacity;
    color.a = color.a * u_opacity;
@@ -111,18 +108,20 @@ var swirlMaterial = game.graphicsContext.createMaterial({
   fragmentSource
 });
 
-var outlineMaterial = game.graphicsContext.createMaterial({
-  name: 'outline',
-  fragmentSource: outline
-})
+
 
 var click = ex.vec(0, 0);
 
 game.input.pointers.primary.on('down', evt => {
   click = evt.worldPos; // might need to change if you have a camera
 });
 
-var actor = new ex.Actor({x: 100, y: 100, width: 50, height: 50});
+var outlineMaterial = game.graphicsContext.createMaterial({
+  name: 'outline',
+  fragmentSource: outline
+})
+
+var actor = new ex.Actor({ x: 100, y: 100, width: 50, height: 50 });
 actor.onInitialize = () => {
   var sprite = new ex.Sprite({
     image: tex,
@@ -132,16 +131,10 @@ actor.onInitialize = () => {
     }
   });
   actor.graphics.add(sprite);
-  actor.graphics.material = outlineMaterial;
 };
+actor.graphics.material = outlineMaterial;
 
-actor.onPostUpdate = (_, delta) => {
-  time += (delta / 1000);
-  outlineMaterial.getShader().use();
-  outlineMaterial.getShader().trySetUniformFloat('iTime', time);
-}
-
-var heartActor = new ex.Actor({x: 200, y: 200});
+var heartActor = new ex.Actor({ x: 200, y: 200 });
 heartActor.onInitialize = () => {
   var sprite = heartImage.toSprite();
   sprite.scale = ex.vec(4, 4);
@@ -153,24 +146,151 @@ game.add(heartActor);
 
 game.input.pointers.primary.on('move', evt => {
   heartActor.pos = evt.worldPos;
-  swirlMaterial.getShader().use();
-  swirlMaterial.getShader().trySetUniformFloatVector('iMouse', evt.worldPos);
+  swirlMaterial.update(shader => {
+    shader.trySetUniformFloatVector('iMouse', evt.worldPos);
+  });
 });
 
 
-var backgroundActor = new ex.ScreenElement({x: 0, y: 0, width: 512, height: 512, z: -1});
-var time = 0;
-backgroundActor.onPostUpdate = (_, delta) => {
-  time += (delta / 1000);
-  // swirlMaterial.getShader().use();
-  // swirlMaterial.getShader().trySetUniformFloat('iTime', time);
-  // swirlMaterial.getShader().trySetUniformFloatVector('iMouse', click);
-}
+var backgroundActor = new ex.ScreenElement({ x: 0, y: 0, width: 512, height: 512, z: -1 });
+
 backgroundActor.onInitialize = () => {
   backgroundActor.graphics.add(background.toSprite());
   backgroundActor.graphics.material = swirlMaterial;
 };
 
+
+// material without graphic!?
+var waterFrag = glsl`#version 300 es
+precision mediump float;
+
+#define NUM_NOISE_OCTAVES 20
+
+// Precision-adjusted variations of https://www.shadertoy.com/view/4djSRW
+float hash(float p) { p = fract(p * 0.011); p *= p + 7.5; p *= p + p; return fract(p); }
+float hash(vec2 p) {vec3 p3 = fract(vec3(p.xyx) * 0.13); p3 += dot(p3, p3.yzx + 3.333); return fract((p3.x + p3.y) * p3.z); }
+
+float noise(float x) {
+    float i = floor(x);
+    float f = fract(x);
+    float u = f * f * (3.0 - 2.0 * f);
+    return mix(hash(i), hash(i + 1.0), u);
+}
+
+
+float noise(vec2 x) {
+    vec2 i = floor(x);
+    vec2 f = fract(x);
+
+	// Four corners in 2D of a tile
+	float a = hash(i);
+    float b = hash(i + vec2(1.0, 0.0));
+    float c = hash(i + vec2(0.0, 1.0));
+    float d = hash(i + vec2(1.0, 1.0));
+
+    // Simple 2D lerp using smoothstep envelope between the values.
+	// return vec3(mix(mix(a, b, smoothstep(0.0, 1.0, f.x)),
+	//			mix(c, d, smoothstep(0.0, 1.0, f.x)),
+	//			smoothstep(0.0, 1.0, f.y)));
+
+	// Same code, with the clamps in smoothstep and common subexpressions
+	// optimized away.
+    vec2 u = f * f * (3.0 - 2.0 * f);
+	return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y;
+}
+
+float fbm(float x) {
+	float v = 0.0;
+	float a = 0.5;
+	float shift = float(100);
+	for (int i = 0; i < NUM_NOISE_OCTAVES; ++i) {
+		v += a * noise(x);
+		x = x * 2.0 + shift;
+		a *= 0.5;
+	}
+	return v;
+}
+
+
+float fbm(vec2 x) {
+	float v = 0.0;
+	float a = 0.5;
+	vec2 shift = vec2(100);
+	// Rotate to reduce axial bias
+    mat2 rot = mat2(cos(0.5), sin(0.5), -sin(0.5), cos(0.50));
+	for (int i = 0; i < NUM_NOISE_OCTAVES; ++i) {
+		v += a * noise(x);
+		x = rot * x * 2.0 + shift;
+		a *= 0.5;
+	}
+	return v;
+}
+
+
+
+uniform float u_time_ms;
+uniform vec4 u_color;
+uniform sampler2D u_graphic;
+uniform sampler2D u_screen_texture;
+
+uniform vec2 u_resolution; // screen resolution
+uniform vec2 u_graphic_resolution; // graphic resolution
+
+in vec2 v_uv;
+in vec2 v_screenuv;
+out vec4 fragColor;
+void main() {
+  float time_sec = u_time_ms / 1000.;
+  float wave_amplitude = .525;
+  float wave_speed = 1.8;
+  float wave_period = .175;
+  vec2 scale = vec2(2.5, 8.5);
+
+  float waves = v_uv.y * scale.y + 
+        sin(v_uv.x * scale.x / wave_period - time_sec * wave_speed) *
+        cos(0.2 * v_uv.x * scale.x /wave_period + time_sec * wave_speed) *
+        wave_amplitude - wave_amplitude;
+
+  
+  float distortion = noise(v_uv*scale*vec2(2.1, 1.05) + time_sec * 0.12) * .25 - .125;
+
+  vec2 reflected_screenuv = vec2(v_screenuv.x - distortion, v_screenuv.y);
+  vec4 screen_color = texture(u_screen_texture, reflected_screenuv);
+
+  vec4 wave_crest_color = vec4(1);
+  float wave_crest = clamp(smoothstep(0.1, 0.14, waves) - smoothstep(0.018, 0.99, waves), 0., 1.);
+
+  fragColor.a = smoothstep(0.1, 0.12, waves);
+  vec3 mixColor = (u_color.rgb * u_color.a); // pre-multiplied alpha
+  
+  fragColor.rgb = mix(screen_color.rgb, mixColor, u_color.a)*fragColor.a + (wave_crest_color.rgb * wave_crest);
+}`;
+
+
+var waterMaterial = game.graphicsContext.createMaterial({
+  name: 'water',
+  fragmentSource: waterFrag,
+  color: ex.Color.fromRGB(55, 0, 200, .6)
+});
+var reflection = new ex.Actor({
+  x: 0,
+  y: game.screen.resolution.height/2,
+  anchor: ex.vec(0, 0),
+  width: 512,
+  height: game.screen.resolution.height/2,
+  coordPlane: ex.CoordPlane.Screen,
+  color: ex.Color.Red
+});
+
+reflection.graphics.material = waterMaterial;
+reflection.z = 99;
+
 game.add(actor);
 game.add(backgroundActor);
-game.start(loader);
+game.add(reflection);
+
+
+game.start(loader).then(async () => {
+  // const image = await game.screenshot(true);
+  // document.body.appendChild(image);
+});

src/engine/Graphics/Context/ExcaliburGraphicsContextWebGL.ts

@@ -130,6 +130,8 @@ export class ExcaliburGraphicsContextWebGL implements ExcaliburGraphicsContext {
 
   public textureLoader: TextureLoader;
 
+  public materialScreenTexture: WebGLTexture;
+
   public get z(): number {
     return this._state.current.z;
   }
@@ -229,6 +231,16 @@ export class ExcaliburGraphicsContextWebGL implements ExcaliburGraphicsContext {
     this.register(new PointRenderer());
     this.register(new LineRenderer());
 
+
+    this.materialScreenTexture = gl.createTexture();
+    gl.bindTexture(gl.TEXTURE_2D, this.materialScreenTexture);
+    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this.width, this.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT);
+    gl.bindTexture(gl.TEXTURE_2D, null);
+
     this._screenRenderer = new ScreenPassPainter(gl);
 
     this._renderTarget = new RenderTarget({
@@ -562,6 +574,13 @@ export class ExcaliburGraphicsContextWebGL implements ExcaliburGraphicsContext {
             currentRenderer = this._renderers.get(currentRendererName);
           }
 
+          // ! hack to grab screen texture before materials run because they might want it
+          if (currentRenderer instanceof MaterialRenderer && this.material.isUsingScreenTexture) {
+            const gl = this.__gl;
+            gl.bindTexture(gl.TEXTURE_2D, this.materialScreenTexture);
+            gl.copyTexImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 0, 0, this.width, this.height, 0);
+            this._renderTarget.use();
+          }
           // If we are still using the same renderer we can add to the current batch
           currentRenderer.draw(...this._drawCalls[i].args);
         }