Merge pull request #281 from mkkellogg/dev

Release 0.4.3
mkkellogg · Jul 15, 2024 · 992ed9a · 992ed9a
2 parents 19e01ad + bbb3b86
commit 992ed9a
Show file tree

Hide file tree

Showing 14 changed files with 1,474 additions and 596 deletions.
diff --git a/README.md b/README.md
@@ -30,12 +30,23 @@ When I started, web-based viewers were already available -- A WebGL-based viewer
 - Custom `.ksplat` file format still needs work, especially around compression
 - The default, integer based splat sort does not work well for larger scenes. In that case a value of `false` for the `integerBasedSort` viewer parameter can force a slower, floating-point based sort
 
+## Limitations
+
+Currently there are limits on the number of splats that can be rendered, and those limits depend mainly on the degree of spherical harmonics desired. Those limits are:
+
+| Spherical harmonics degree | Max splat count
+| --- | ---
+| `0` | ~ 16,000,000
+| `1` | ~ 11,000,000
+| `2` | ~ 8,000,000
+
+Future work will include optimizing how splat data is packed into data textures, which will help increase these limits.
+
 ## Future work
 This is still very much a work in progress! There are several things that still need to be done:
-  - Improve the method by which splat data is stored in textures
+  - Improve the way splat data is packed into data textures
   - Continue optimizing CPU-based splat sort - maybe try an incremental sort of some kind?
-  - Add editing mode, allowing users to modify scene and export changes
-  - Support very large scenes
+  - Support very large scenes (streaming sections & LOD)
 
 ## Online demo
 [https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php](https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php)
@@ -318,6 +329,7 @@ Advanced `Viewer` parameters
 | `enableOptionalEffects` | When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment. Default is `false` for performance reasons. These properties are separate from transform properties (scale, rotation, position) that are enabled by the `dynamicScene` parameter.
 | `plyInMemoryCompressionLevel` | Level to compress `.ply` files when loading them for direct rendering (not exporting to `.ksplat`). Valid values are the same as `.ksplat` compression levels (0, 1, or 2). Default is 2.
 | `freeIntermediateSplatData` | When true, the intermediate splat data that is the result of decompressing splat bufffer(s) and used to populate data textures will be freed. This will reduces memory usage, but if that data needs to be modified it will need to be re-populated from the splat buffer(s). Defaults to `false`.
+| `splatRenderMode` | Determine which splat rendering mode to enable. Valid values are defined in the `SplatRenderMode` enum: `ThreeD` and `TwoD`. `ThreeD` is the original/traditional mode and `TwoD` is the new mode described here: https://surfsplatting.github.io/
 <br>
 
 ### Creating KSPLAT files

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -4,7 +4,7 @@
         "type": "git",
         "url": "https://github.com/mkkellogg/GaussianSplats3D"
     },
-    "version": "0.4.2",
+    "version": "0.4.3",
     "description": "Three.js-based 3D Gaussian splat viewer",
     "module": "build/gaussian-splats-3d.module.js",
     "main": "build/gaussian-splats-3d.umd.cjs",

diff --git a/src/DropInViewer.js b/src/DropInViewer.js
@@ -20,11 +20,26 @@ export class DropInViewer extends THREE.Group {
 
         this.viewer = new Viewer(options);
         this.splatMesh = null;
+        this.updateSplatMesh();
 
         this.callbackMesh = DropInViewer.createCallbackMesh();
         this.add(this.callbackMesh);
         this.callbackMesh.onBeforeRender = DropInViewer.onBeforeRender.bind(this, this.viewer);
 
+        this.viewer.onSplatMeshChanged(() => {
+            this.updateSplatMesh();
+        });
+
+    }
+
+    updateSplatMesh() {
+        if (this.splatMesh !== this.viewer.splatMesh) {
+            if (this.splatMesh) {
+                this.remove(this.splatMesh);
+            }
+            this.splatMesh = this.viewer.splatMesh;
+            this.add(this.viewer.splatMesh);
+        }
     }
 
     /**
@@ -85,22 +100,19 @@ export class DropInViewer extends THREE.Group {
         return this.viewer.getSplatScene(sceneIndex);
     }
 
-    removeSplatScene(index) {
-        return this.viewer.removeSplatScene(index);
+    removeSplatScene(index, showLoadingUI = true) {
+        return this.viewer.removeSplatScene(index, showLoadingUI);
+    }
+
+    removeSplatScenes(indexes, showLoadingUI = true) {
+        return this.viewer.removeSplatScenes(indexes, showLoadingUI);
     }
 
     dispose() {
         return this.viewer.dispose();
     }
 
     static onBeforeRender(viewer, renderer, threeScene, camera) {
-        if (this.splatMesh !== this.viewer.splatMesh) {
-            if (this.splatMesh) {
-                this.remove(this.splatMesh);
-            }
-            this.splatMesh = this.viewer.splatMesh;
-            this.add(this.viewer.splatMesh);
-        }
         viewer.update(renderer, camera);
     }
 

diff --git a/src/SplatRenderMode.js b/src/SplatRenderMode.js
@@ -0,0 +1,4 @@
+export const SplatRenderMode = {
+    ThreeD: 0,
+    TwoD: 1
+};
diff --git a/src/Viewer.js b/src/Viewer.js
@@ -23,6 +23,7 @@ import { LoaderStatus } from './loaders/LoaderStatus.js';
 import { RenderMode } from './RenderMode.js';
 import { LogLevel } from './LogLevel.js';
 import { SceneRevealMode } from './SceneRevealMode.js';
+import { SplatRenderMode } from './SplatRenderMode.js';
 
 const THREE_CAMERA_FOV = 50;
 const MINIMUM_DISTANCE_TO_NEW_FOCAL_POINT = .75;
@@ -142,7 +143,7 @@ export class Viewer {
         this.logLevel = options.logLevel || LogLevel.None;
 
         // Degree of spherical harmonics to utilize in rendering splats (assuming the data is present in the splat scene).
-        // Valid values are 0 - 3. Default value is 0.
+        // Valid values are 0 - 2. Default value is 0.
         this.sphericalHarmonicsDegree = options.sphericalHarmonicsDegree || 0;
 
         // When true, allows for usage of extra properties and attributes during rendering for effects such as opacity adjustment.
@@ -180,6 +181,13 @@ export class Viewer {
             }
         }
 
+        // Tell the viewer how to render the splats
+        if (options.splatRenderMode === undefined || options.splatRenderMode === null) {
+            options.splatRenderMode = SplatRenderMode.ThreeD;
+        }
+        this.splatRenderMode = options.splatRenderMode;
+
+        this.onSplatMeshChangedCallback = null;
         this.createSplatMesh();
 
         this.controls = null;
@@ -249,14 +257,17 @@ export class Viewer {
         this.initialized = false;
         this.disposing = false;
         this.disposed = false;
+        this.disposePromise = null;
         if (!this.dropInMode) this.init();
     }
 
     createSplatMesh() {
-        this.splatMesh = new SplatMesh(this.dynamicScene, this.enableOptionalEffects, this.halfPrecisionCovariancesOnGPU,
-                                       this.devicePixelRatio, this.gpuAcceleratedSort, this.integerBasedSort, this.antialiased,
-                                       this.maxScreenSpaceSplatSize, this.logLevel, this.sphericalHarmonicsDegree);
+        this.splatMesh = new SplatMesh(this.splatRenderMode, this.dynamicScene, this.enableOptionalEffects,
+                                       this.halfPrecisionCovariancesOnGPU, this.devicePixelRatio, this.gpuAcceleratedSort,
+                                       this.integerBasedSort, this.antialiased, this.maxScreenSpaceSplatSize, this.logLevel,
+                                       this.sphericalHarmonicsDegree);
         this.splatMesh.frustumCulled = false;
+        if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback();
     }
 
     init() {
@@ -366,7 +377,7 @@ export class Viewer {
                     this.perspectiveControls = new OrbitControls(this.camera, this.renderer.domElement);
                 }
             }
-            for (let controls of [this.perspectiveControls, this.orthographicControls]) {
+            for (let controls of [this.orthographicControls, this.perspectiveControls,]) {
                 if (controls) {
                     controls.listenToKeyEvents(window);
                     controls.rotateSpeed = 0.5;
@@ -375,9 +386,11 @@ export class Viewer {
                     controls.enableDamping = true;
                     controls.dampingFactor = 0.05;
                     controls.target.copy(this.initialCameraLookAt);
+                    controls.update();
                 }
             }
             this.controls = this.camera.isOrthographicCamera ? this.orthographicControls : this.perspectiveControls;
+            this.controls.update();
         }
     }
 
@@ -411,6 +424,10 @@ export class Viewer {
         this.renderMode = renderMode;
     }
 
+    onSplatMeshChanged(callback) {
+        this.onSplatMeshChangedCallback = callback;
+    }
+
     onKeyDown = function() {
 
         const forward = new THREE.Vector3();
@@ -1021,7 +1038,8 @@ export class Viewer {
             return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt,
                                          splatAlphaRemovalThreshold, this.plyInMemoryCompressionLevel, this.sphericalHarmonicsDegree);
         }
-        return AbortablePromise.reject(new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`));
+
+        throw new Error(`Viewer::downloadSplatSceneToSplatBuffer -> File format not supported: ${path}`);
     }
 
     static isProgressivelyLoadable(format) {
@@ -1043,6 +1061,13 @@ export class Viewer {
             this.splatRenderReady = false;
             let splatProcessingTaskId = null;
 
+            const removeSplatProcessingTask = () => {
+                if (splatProcessingTaskId !== null) {
+                    this.loadingSpinner.removeTask(splatProcessingTaskId);
+                    splatProcessingTaskId = null;
+                }
+            };
+
             const finish = (buildResults, resolver) => {
                 if (this.isDisposingOrDisposed()) return;
 
@@ -1062,21 +1087,23 @@ export class Viewer {
 
                 this.updateSplatSort(true);
 
-                if (enableRenderBeforeFirstSort) {
+                if (!this.sortWorker) {
                     this.splatRenderReady = true;
+                    removeSplatProcessingTask();
+                    resolver();
                 } else {
-                    this.runAfterNextSort.push(() => {
+                    if (enableRenderBeforeFirstSort) {
                         this.splatRenderReady = true;
+                    } else {
+                        this.runAfterNextSort.push(() => {
+                            this.splatRenderReady = true;
+                        });
+                    }
+                    this.runAfterNextSort.push(() => {
+                        removeSplatProcessingTask();
+                        resolver();
                     });
                 }
-
-                this.runAfterNextSort.push(() => {
-                    if (splatProcessingTaskId !== null) {
-                        this.loadingSpinner.removeTask(splatProcessingTaskId);
-                        splatProcessingTaskId = null;
-                    }
-                    resolver();
-                });
             };
 
             return new Promise((resolve) => {
@@ -1245,7 +1272,11 @@ export class Viewer {
         this.sortRunning = false;
     }
 
-    removeSplatScene(index, showLoadingUI = true) {
+    removeSplatScene(indexToRemove, showLoadingUI = true) {
+        return this.removeSplatScenes([indexToRemove], showLoadingUI);
+    }
+
+    removeSplatScenes(indexesToRemove, showLoadingUI = true) {
         if (this.isLoadingOrUnloading()) {
             throw new Error('Cannot remove splat scene while another load or unload is already in progress.');
         }
@@ -1294,7 +1325,14 @@ export class Viewer {
                 const savedSceneOptions = [];
                 const savedSceneTransformComponents = [];
                 for (let i = 0; i < this.splatMesh.scenes.length; i++) {
-                    if (i !== index) {
+                    let shouldRemove = false;
+                    for (let indexToRemove of indexesToRemove) {
+                        if (indexToRemove === i) {
+                            shouldRemove = true;
+                            break;
+                        }
+                    }
+                    if (!shouldRemove) {
                         const scene = this.splatMesh.scenes[i];
                         savedSplatBuffers.push(scene.splatBuffer);
                         savedSceneOptions.push(this.splatMesh.sceneOptions[i]);
@@ -1307,6 +1345,7 @@ export class Viewer {
                 }
                 this.disposeSortWorker();
                 this.splatMesh.dispose();
+                this.sceneRevealMode = SceneRevealMode.Instant;
                 this.createSplatMesh();
                 this.addSplatBuffers(savedSplatBuffers, savedSceneOptions, true, false, true)
                 .then(() => {
@@ -1373,7 +1412,8 @@ export class Viewer {
      * Dispose of all resources held directly and indirectly by this viewer.
      */
     async dispose() {
-        this.disposing = true;
+        if (this.isDisposingOrDisposed()) return this.disposePromise;
+
         let waitPromises = [];
         let promisesToAbort = [];
         for (let promiseKey in this.splatSceneDownloadPromises) {
@@ -1386,7 +1426,9 @@ export class Viewer {
         if (this.sortPromise) {
             waitPromises.push(this.sortPromise);
         }
-        const disposePromise = Promise.all(waitPromises).finally(() => {
+
+        this.disposing = true;
+        this.disposePromise = Promise.all(waitPromises).finally(() => {
             this.stop();
             if (this.controls) {
                 this.controls.dispose();
@@ -1439,7 +1481,7 @@ export class Viewer {
         promisesToAbort.forEach((toAbort) => {
             toAbort.abort('Scene disposed');
         });
-        return disposePromise;
+        return this.disposePromise;
     }
 
     selfDrivenUpdate() {
@@ -1756,7 +1798,10 @@ export class Viewer {
 
         return async function(force = false) {
             if (this.sortRunning) return;
-            if (this.splatMesh.getSplatCount() <= 0) return;
+            if (this.splatMesh.getSplatCount() <= 0) {
+                this.splatRenderCount = 0;
+                return;
+            }
 
             let angleDiff = 0;
             let positionDiff = 0;

diff --git a/src/index.js b/src/index.js
@@ -17,6 +17,7 @@ import { WebXRMode } from './webxr/WebXRMode.js';
 import { RenderMode } from './RenderMode.js';
 import { LogLevel } from './LogLevel.js';
 import { SceneRevealMode } from './SceneRevealMode.js';
+import { SplatRenderMode } from './SplatRenderMode.js';
 
 export {
     PlyParser,
@@ -37,5 +38,6 @@ export {
     WebXRMode,
     RenderMode,
     LogLevel,
-    SceneRevealMode
+    SceneRevealMode,
+    SplatRenderMode
 };