fix: Issue with raycasting complex collision groups (#2842)

- Fixed issue where raycasting with more complex collision groups was not working as expected
- Added new feature to collision group raycasting, directly provide a `collisionMask` that you want to search for. Theoretically this is less confusing

```typescript
const playerGroup = ex.CollisionGroupManager.create('playerGroup');
const notPlayersMask = ~playersGroup.category;
const hits = engine.currentScene.physics.rayCast(
  new ex.Ray(player.pos, playerDir),
  {
    maxDistance: playerSightDistance,
    // Search for all categories that match the mask
    collisionMask: notPlayers,
    searchAllColliders: false
  });
```

CHANGELOG.md

@@ -15,6 +15,50 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Added new feature to collision group raycasting, directly provide a `collisionMask` that you want to search for.
+
+```typescript
+const playerGroup = ex.CollisionGroupManager.create('playerGroup');
+const notPlayersMask = ~playersGroup.category;
+const hits = engine.currentScene.physics.rayCast(
+  new ex.Ray(player.pos, playerDir),
+  {
+    maxDistance: playerSightDistance,
+    // Search for all categories that match the mask
+    collisionMask: notPlayers,
+    searchAllColliders: false
+  });
+```
+
+
+### Fixed
+
+- Fixed issue where raycasting with more complex collision groups was not working as expected
+
+### Updates
+
+- 
+
+### Changed
+
+- 
+
+<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
+<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
+<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
+
+## [v0.28.2]
+
+### Breaking Changes
+
+-
+
+### Deprecated
+
+-
+
+### Added
+
 - Added `ex.Engine.version` to report the current excalibur version build string
 - Added new `ex.Screen.events`
   - `screen.events.on('resize', (evt) => )` Will emit when the screen is resized
@@ -47,10 +91,6 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Changed the canvas 2d fallback default, no longer is enabled by default. Developers must opt in.
 - Allow entity names to be set after construction! Entities will now default to a name "Entity#1234" followed by an id.
 
-<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
-<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
-<!--------------------------------- DO NOT EDIT BELOW THIS LINE --------------------------------->
-
 ## [v0.28.0]
 
 ### Breaking Changes

sandbox/tests/raycast/main.ts

@@ -5,13 +5,18 @@ var game = new ex.Engine({
   displayMode: ex.DisplayMode.FitScreenAndFill
 });
 var random = new ex.Random(1337);
+// collides with everything but players
+var playerGroup = ex.CollisionGroupManager.create('playerGroup');
 var blockGroup = ex.CollisionGroupManager.create('blockGroup');
+var notPlayers = ~playerGroup.category;
+var notPlayers2 = playerGroup.invert();
 
 var player = new ex.Actor({
   name: 'player',
   pos: ex.vec(100, 100),
   width: 40,
   height: 40,
+  collisionGroup: playerGroup,
   color: ex.Color.Red,
   z: 10
 });
@@ -39,7 +44,8 @@ player.onPostUpdate = (engine) => {
     new ex.Ray(player.pos, playerDir),
     {
       maxDistance: playerSightDistance,
-      collisionGroup: blockGroup,
+      // collisionMask: notPlayers,
+      collisionGroup: notPlayers2,
       searchAllColliders: false
     });
 

src/engine/Collision/Detection/DynamicTreeCollisionProcessor.ts

@@ -40,9 +40,13 @@ export interface RayCastOptions {
    */
   maxDistance?: number;
   /**
-   * Optionally specify a collision group to consider in the ray cast, default is All
+   * Optionally specify a collision group to target in the ray cast, default is All.
    */
   collisionGroup?: CollisionGroup;
+  /**
+   * Optionally specify a collision mask to target multiple collision categories
+   */
+  collisionMask?: number;
   /**
    * Optionally specify to search for all colliders that intersect the ray cast, not just the first which is the default
    */
@@ -67,13 +71,17 @@ export class DynamicTreeCollisionProcessor implements CollisionProcessor {
   public rayCast(ray: Ray, options?: RayCastOptions): RayCastHit[] {
     const results: RayCastHit[] = [];
     const maxDistance = options?.maxDistance ?? Infinity;
-    const collisionGroup = options?.collisionGroup ?? CollisionGroup.All;
+    const collisionGroup = options?.collisionGroup;
+    const collisionMask = !collisionGroup ? options?.collisionMask ?? CollisionGroup.All.category : collisionGroup.category;
     const searchAllColliders = options?.searchAllColliders ?? false;
     this._dynamicCollisionTree.rayCastQuery(ray, maxDistance, (collider) => {
       const owner = collider.owner;
       const maybeBody = owner.get(BodyComponent);
+
+      const canCollide = (collisionMask & maybeBody.group.category) !== 0;
+
       // Early exit if not the right group
-      if (collisionGroup.mask !== CollisionGroup.All.mask && maybeBody?.group?.mask !== collisionGroup.mask) {
+      if (maybeBody?.group && !canCollide) {
         return false;
       }
 

src/engine/Collision/Group/CollisionGroup.ts

@@ -1,3 +1,5 @@
+import { CollisionGroupManager } from './CollisionGroupManager';
+
 /**
  * CollisionGroups indicate like members that do not collide with each other. Use [[CollisionGroupManager]] to create [[CollisionGroup]]s
  *
@@ -88,10 +90,14 @@ export class CollisionGroup {
 
   /**
    * Evaluates whether 2 collision groups can collide
+   *
+   * This means the mask has the same bit set the other category and vice versa
    * @param other  CollisionGroup
    */
   public canCollide(other: CollisionGroup): boolean {
-    return (this.category & other.mask) !== 0 && (other.category & this.mask) !== 0;
+    const overlap1 = this.category & other.mask;
+    const overlap2 = this.mask & other.category;
+    return (overlap1 !== 0) && (overlap2 !== 0);
   }
 
   /**
@@ -100,7 +106,9 @@ export class CollisionGroup {
    * @returns CollisionGroup
    */
   public invert(): CollisionGroup {
-    return new CollisionGroup('~(' + this.name + ')', ~this.category, ~this.mask);
+    const group = CollisionGroupManager.create('~(' + this.name + ')', ~this.mask | 0);
+    group._category = ~this.category;
+    return group;
   }
 
   /**
@@ -112,14 +120,23 @@ export class CollisionGroup {
     const combinedCategory = collisionGroups.reduce((current, g) => g.category | current, 0b0);
     const combinedMask = ~combinedCategory;
 
-    return new CollisionGroup(combinedName, combinedCategory, combinedMask);
+    return CollisionGroupManager.create(combinedName, combinedMask);
   }
 
   /**
    * Creates a collision group that collides with the listed groups
    * @param collisionGroups
    */
   public static collidesWith(collisionGroups: CollisionGroup[]) {
-    return CollisionGroup.combine(collisionGroups).invert();
+    const combinedName = `collidesWith(${collisionGroups.map((c) => c.name).join('+')})`;
+    const combinedMask = collisionGroups.reduce((current, g) => g.category | current, 0b0);
+    return CollisionGroupManager.create(combinedName, combinedMask);
+  }
+
+  public toString() {
+    return `
+category: ${this.category.toString(2).padStart(32, '0')}
+mask:     ${(this.mask>>>0).toString(2).padStart(32, '0')}
+    `;
   }
 }

src/engine/Collision/Group/CollisionGroupManager.ts

@@ -21,7 +21,11 @@ export class CollisionGroupManager {
       throw new Error(`Cannot have more than ${this._MAX_GROUPS} collision groups`);
     }
     if (this._GROUPS.get(name)) {
-      throw new Error(`Collision group ${name} already exists`);
+      const existingGroup = this._GROUPS.get(name);
+      if (existingGroup.mask === mask) {
+        return existingGroup;
+      }
+      throw new Error(`Collision group ${name} already exists with a different mask!`);
     }
     const group = new CollisionGroup(name, this._CURRENT_BIT, mask !== undefined ? mask : ~this._CURRENT_BIT);
     this._CURRENT_BIT = (this._CURRENT_BIT << 1) | 0;

src/spec/CollisionGroupSpec.ts

@@ -33,7 +33,12 @@ describe('A Collision Group', () => {
 
   it('can invert collision groups', () => {
     const invertedA = groupA.invert();
+    expect(invertedA.category).toBe(~groupA.category);
+    expect(invertedA.mask).toBe(~groupA.mask);
     expect(invertedA.name).toBe('~(groupA)');
+    expect(groupA.canCollide(groupA)).toBe(false);
+    expect(groupA.canCollide(groupB)).toBe(true);
+    expect(groupA.canCollide(groupC)).toBe(true);
     expect(invertedA.canCollide(groupA)).toBe(true);
     expect(invertedA.canCollide(groupB)).toBe(false);
     expect(invertedA.canCollide(groupC)).toBe(false);
@@ -49,7 +54,7 @@ describe('A Collision Group', () => {
 
   it('can create collidesWith groups', () => {
     const collidesWithBAndC = ex.CollisionGroup.collidesWith([groupB, groupC]);
-    expect(collidesWithBAndC.name).toBe('~(groupB+groupC)');
+    expect(collidesWithBAndC.name).toBe('collidesWith(groupB+groupC)');
     expect(collidesWithBAndC.canCollide(groupA)).toBe(false);
     expect(collidesWithBAndC.canCollide(groupB)).toBe(true);
     expect(collidesWithBAndC.canCollide(groupC)).toBe(true);
@@ -80,10 +85,10 @@ describe('A Collision Group', () => {
 
   it('should throw if 2 groups of the same name are created', () => {
     ex.CollisionGroupManager.reset();
-    const maybeA = ex.CollisionGroupManager.create('A');
+    const maybeA = ex.CollisionGroupManager.create('A', 0b1);
     expect(() => {
-      const maybeAAlso = ex.CollisionGroupManager.create('A');
-    }).toThrowError('Collision group A already exists');
+      const maybeAAlso = ex.CollisionGroupManager.create('A', 0b10);
+    }).toThrowError('Collision group A already exists with a different mask!');
   });
 
   it('should allow 32 collision groups', () => {