diff --git a/README.md b/README.md index 690e85a..454c68d 100644 --- a/README.md +++ b/README.md @@ -334,6 +334,7 @@ Advanced `Viewer` parameters | `renderMode` | Controls when the viewer renders the scene. Valid values are defined in the `RenderMode` enum: `Always`, `OnChange`, and `Never`. Defaults to `Always`. | `sceneRevealMode` | Controls the fade-in effect used when the scene is loaded. Valid values are defined in the `SceneRevealMode` enum: `Default`, `Gradual`, and `Instant`. `Default` results in a nice, slow fade-in effect for progressively loaded scenes, and a fast fade-in for non progressively loaded scenes. `Gradual` will force a slow fade-in for all scenes. `Instant` will force all loaded scene data to be immediately visible. | `antialiased` | When true, will perform additional steps during rendering to address artifacts caused by the rendering of gaussians at substantially different resolutions than that at which they were rendered during training. This will only work correctly for models that were trained using a process that utilizes this compensation calculation. For more details: https://github.com/nerfstudio-project/gsplat/pull/117, https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093 +| `kernel2DSize` | A constant added to the size of the 2D screen-space gaussian kernel used in rendering splats. Default value is 0.3. | `focalAdjustment` | Hacky, non-scientific parameter for tweaking focal length related calculations. For scenes with very small gaussians & small details, increasing this value can help improve visual quality. Default value is 1.0. | `logLevel` | Verbosity of the console logging. Defaults to `GaussianSplats3D.LogLevel.None`. | `sphericalHarmonicsDegree` | Degree of spherical harmonics to utilize in rendering splats (assuming the data is present in the splat scene). Valid values are 0, 1, or 2. Default value is 0. @@ -361,7 +362,8 @@ GaussianSplats3D.PlyLoader.loadFromURL('', minimumAlpha, compressionLevel, optimizeSplatData, - sphericalHarmonicsDegree) + sphericalHarmonicsDegree, + headers) .then((splatBuffer) => { GaussianSplats3D.KSplatLoader.downloadFile(splatBuffer, 'converted_file.ksplat'); }); diff --git a/package-lock.json b/package-lock.json index decfc9e..0369b1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.5", + "version": "0.4.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mkkellogg/gaussian-splats-3d", - "version": "0.4.5", + "version": "0.4.6", "license": "MIT", "devDependencies": { "@babel/core": "7.22.0", diff --git a/package.json b/package.json index ff08960..003fa6e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "type": "git", "url": "https://github.com/mkkellogg/GaussianSplats3D" }, - "version": "0.4.5", + "version": "0.4.6", "description": "Three.js-based 3D Gaussian splat viewer", "module": "build/gaussian-splats-3d.module.js", "main": "build/gaussian-splats-3d.umd.cjs", diff --git a/src/DropInViewer.js b/src/DropInViewer.js index 5db045f..342e1b1 100644 --- a/src/DropInViewer.js +++ b/src/DropInViewer.js @@ -13,7 +13,6 @@ export class DropInViewer extends THREE.Group { options.selfDrivenMode = false; options.useBuiltInControls = false; options.rootElement = null; - options.ignoreDevicePixelRatio = false; options.dropInMode = true; options.camera = undefined; options.renderer = undefined; diff --git a/src/Util.js b/src/Util.js index 870dbe6..29c71b4 100644 --- a/src/Util.js +++ b/src/Util.js @@ -54,7 +54,7 @@ export const rgbaArrayToInteger = function(arr, offset) { return arr[offset] + (arr[offset + 1] << 8) + (arr[offset + 2] << 16) + (arr[offset + 3] << 24); }; -export const fetchWithProgress = function(path, onProgress, saveChunks = true) { +export const fetchWithProgress = function(path, onProgress, saveChunks = true, headers) { const abortController = new AbortController(); const signal = abortController.signal; @@ -65,7 +65,9 @@ export const fetchWithProgress = function(path, onProgress, saveChunks = true) { }; return new AbortablePromise((resolve, reject) => { - fetch(path, { signal }) + const fetchOptions = { signal }; + if (headers) fetchOptions.headers = headers; + fetch(path, fetchOptions) .then(async (data) => { // Handle error conditions where data is still returned if (!data.ok) { diff --git a/src/Viewer.js b/src/Viewer.js index 548fc01..b9ff018 100644 --- a/src/Viewer.js +++ b/src/Viewer.js @@ -72,7 +72,7 @@ export class Viewer { // Tells the viewer to pretend the device pixel ratio is 1, which can boost performance on devices where it is larger, // at a small cost to visual quality this.ignoreDevicePixelRatio = options.ignoreDevicePixelRatio || false; - this.devicePixelRatio = this.ignoreDevicePixelRatio ? 1 : window.devicePixelRatio; + this.devicePixelRatio = this.ignoreDevicePixelRatio ? 1 : (window.devicePixelRatio || 1); // Tells the viewer to use 16-bit floating point values when storing splat covariance data in textures, instead of 32-bit this.halfPrecisionCovariancesOnGPU = options.halfPrecisionCovariancesOnGPU || false; @@ -117,6 +117,9 @@ export class Viewer { // https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093 this.antialiased = options.antialiased || false; + // This constant is added to the projected 2D screen-space splat scales + this.kernel2DSize = (options.kernel2DSize === undefined) ? 0.3 : options.kernel2DSize; + this.webXRMode = options.webXRMode || WebXRMode.None; if (this.webXRMode !== WebXRMode.None) { this.gpuAcceleratedSort = false; @@ -286,7 +289,7 @@ export class Viewer { this.splatMesh = new SplatMesh(this.splatRenderMode, this.dynamicScene, this.enableOptionalEffects, this.halfPrecisionCovariancesOnGPU, this.devicePixelRatio, this.gpuAcceleratedSort, this.integerBasedSort, this.antialiased, this.maxScreenSpaceSplatSize, this.logLevel, - this.sphericalHarmonicsDegree, this.sceneFadeInRateMultiplier); + this.sphericalHarmonicsDegree, this.sceneFadeInRateMultiplier, this.kernel2DSize); this.splatMesh.frustumCulled = false; if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback(); } @@ -303,7 +306,7 @@ export class Viewer { this.rootElement.style.position = 'absolute'; document.body.appendChild(this.rootElement); } else { - this.rootElement = this.renderer.domElement.parentElement || document.body; + this.rootElement = this.renderer.domElement || document.body; } } @@ -725,6 +728,7 @@ export class Viewer { * * onProgress: Function to be called as file data are received, or other processing occurs * + * headers: Optional HTTP headers to be sent along with splat requests * } * @return {AbortablePromise} */ @@ -819,7 +823,8 @@ export class Viewer { const loadFunc = progressiveLoad ? this.downloadAndBuildSingleSplatSceneProgressiveLoad.bind(this) : this.downloadAndBuildSingleSplatSceneStandardLoad.bind(this); - return loadFunc(path, format, options.splatAlphaRemovalThreshold, buildSection.bind(this), onProgress, hideLoadingUI.bind(this)); + return loadFunc(path, format, options.splatAlphaRemovalThreshold, buildSection.bind(this), + onProgress, hideLoadingUI.bind(this), options.headers); } /** @@ -832,12 +837,13 @@ export class Viewer { * @param {function} buildFunc Function to build the viewer's splat mesh with the downloaded splat buffer * @param {function} onProgress Function to be called as file data are received, or other processing occurs * @param {function} onException Function to be called when exception occurs + * @param {object} headers Optional HTTP headers to pass to use for downloading splat scene * @return {AbortablePromise} */ - downloadAndBuildSingleSplatSceneStandardLoad(path, format, splatAlphaRemovalThreshold, buildFunc, onProgress, onException) { + downloadAndBuildSingleSplatSceneStandardLoad(path, format, splatAlphaRemovalThreshold, buildFunc, onProgress, onException, headers) { - const downloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, - onProgress, false, undefined, format); + const downloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onProgress, false, + undefined, format, headers); const downloadAndBuildPromise = abortablePromiseWithExtractedComponents(downloadPromise.abortHandler); downloadPromise.then((splatBuffer) => { @@ -871,10 +877,11 @@ export class Viewer { * @param {function} buildFunc Function to rebuild the viewer's splat mesh after a new splat buffer section is downloaded * @param {function} onDownloadProgress Function to be called as file data are received * @param {function} onDownloadException Function to be called when exception occurs at any point during the full download + * @param {object} headers Optional HTTP headers to pass to use for downloading splat scene * @return {AbortablePromise} */ downloadAndBuildSingleSplatSceneProgressiveLoad(path, format, splatAlphaRemovalThreshold, buildFunc, - onDownloadProgress, onDownloadException) { + onDownloadProgress, onDownloadException, headers) { let progressiveLoadedSectionBuildCount = 0; let progressiveLoadedSectionBuilding = false; const queuedProgressiveLoadSectionBuilds = []; @@ -917,7 +924,7 @@ export class Viewer { }; const splatSceneDownloadPromise = this.downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold, onDownloadProgress, true, - onProgressiveLoadSectionProgress, format); + onProgressiveLoadSectionProgress, format, headers); const progressiveLoadFirstSectionBuildPromise = abortablePromiseWithExtractedComponents(splatSceneDownloadPromise.abortHandler); const splatSceneDownloadAndBuildPromise = abortablePromiseWithExtractedComponents(); @@ -953,6 +960,11 @@ export class Viewer { * rotation (Array): Rotation of the scene represented as a quaternion, defaults to [0, 0, 0, 1] * * scale (Array): Scene's scale, defaults to [1, 1, 1] + * + * headers: Optional HTTP headers to be sent along with splat requests + * + * format (SceneFormat) Optional, the format of the scene data (.ply, .ksplat, .splat). If not present, the + * file extension in 'path' will be used to determine the format (if it is present) * } * @param {boolean} showLoadingUI Display a loading spinner while the scene is loading, defaults to true * @param {function} onProgress Function to be called as file data are received @@ -998,7 +1010,8 @@ export class Viewer { const options = sceneOptions[i]; const format = (options.format !== undefined && options.format !== null) ? options.format : sceneFormatFromPath(options.path); const baseDownloadPromise = this.downloadSplatSceneToSplatBuffer(options.path, options.splatAlphaRemovalThreshold, - onLoadProgress.bind(this, i), false, undefined, format); + onLoadProgress.bind(this, i), false, undefined, + format, options.headers); baseDownloadPromises.push(baseDownloadPromise); nativeDownloadPromises.push(baseDownloadPromise.promise); } @@ -1044,23 +1057,22 @@ export class Viewer { * @param {boolean} progressiveBuild Construct file sections into splat buffers as they are downloaded * @param {function} onSectionBuilt Function to be called when new section is added to the file * @param {string} format File format of the scene + * @param {object} headers Optional HTTP headers to pass to use for downloading splat scene * @return {AbortablePromise} */ downloadSplatSceneToSplatBuffer(path, splatAlphaRemovalThreshold = 1, onProgress = undefined, - progressiveBuild = false, onSectionBuilt = undefined, format) { + progressiveBuild = false, onSectionBuilt = undefined, format, headers) { const optimizeSplatData = progressiveBuild ? false : this.optimizeSplatData; try { if (format === SceneFormat.Splat) { - return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, - onSectionBuilt, splatAlphaRemovalThreshold, - this.inMemoryCompressionLevel, optimizeSplatData); + return SplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, splatAlphaRemovalThreshold, + this.inMemoryCompressionLevel, optimizeSplatData, headers); } else if (format === SceneFormat.KSplat) { - return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt); + return KSplatLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, headers); } else if (format === SceneFormat.Ply) { - return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, - splatAlphaRemovalThreshold, this.inMemoryCompressionLevel, - optimizeSplatData, this.sphericalHarmonicsDegree); + return PlyLoader.loadFromURL(path, onProgress, progressiveBuild, onSectionBuilt, splatAlphaRemovalThreshold, + this.inMemoryCompressionLevel, optimizeSplatData, this.sphericalHarmonicsDegree, headers); } } catch (e) { if (e instanceof DirectLoadError) { @@ -1870,7 +1882,7 @@ export class Viewer { mvpMatrix.copy(this.camera.matrixWorld).invert(); const mvpCamera = this.perspectiveCamera || this.camera; mvpMatrix.premultiply(mvpCamera.projectionMatrix); - mvpMatrix.multiply(this.splatMesh.matrixWorld); + if (!this.splatMesh.dynamicMode) mvpMatrix.multiply(this.splatMesh.matrixWorld); let gpuAcceleratedSortPromise = Promise.resolve(true); if (this.gpuAcceleratedSort && (queuedSorts.length <= 1 || queuedSorts.length % 2 === 0)) { @@ -1979,7 +1991,7 @@ export class Viewer { if (splatTree) { baseModelView.copy(this.camera.matrixWorld).invert(); - baseModelView.multiply(this.splatMesh.matrixWorld); + if (!this.splatMesh.dynamicMode) baseModelView.multiply(this.splatMesh.matrixWorld); let nodeRenderCount = 0; let splatRenderCount = 0; diff --git a/src/loaders/ksplat/KSplatLoader.js b/src/loaders/ksplat/KSplatLoader.js index 95973ca..77fae01 100644 --- a/src/loaders/ksplat/KSplatLoader.js +++ b/src/loaders/ksplat/KSplatLoader.js @@ -19,7 +19,7 @@ export class KSplatLoader { } }; - static loadFromURL(fileName, externalOnProgress, loadDirectoToSplatBuffer, onSectionBuilt) { + static loadFromURL(fileName, externalOnProgress, loadDirectoToSplatBuffer, onSectionBuilt, headers) { let directLoadBuffer; let directLoadSplatBuffer; @@ -196,7 +196,7 @@ export class KSplatLoader { } }; - return fetchWithProgress(fileName, localOnProgress, !loadDirectoToSplatBuffer).then((fullBuffer) => { + return fetchWithProgress(fileName, localOnProgress, !loadDirectoToSplatBuffer, headers).then((fullBuffer) => { if (externalOnProgress) externalOnProgress(0, '0%', LoaderStatus.Processing); const loadPromise = loadDirectoToSplatBuffer ? directLoadPromise.promise : KSplatLoader.loadFromFileData(fullBuffer); return loadPromise.then((splatBuffer) => { diff --git a/src/loaders/ply/PlyLoader.js b/src/loaders/ply/PlyLoader.js index 4dabc9b..30e415e 100644 --- a/src/loaders/ply/PlyLoader.js +++ b/src/loaders/ply/PlyLoader.js @@ -43,8 +43,9 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class PlyLoader { - static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, - optimizeSplatData = true, outSphericalHarmonicsDegree = 0, sectionSize, sceneCenter, blockSize, bucketSize) { + static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, + minimumAlpha, compressionLevel, optimizeSplatData = true, outSphericalHarmonicsDegree = 0, + headers, sectionSize, sceneCenter, blockSize, bucketSize) { let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; @@ -256,7 +257,7 @@ export class PlyLoader { }; if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); - return fetchWithProgress(fileName, localOnProgress, false).then(() => { + return fetchWithProgress(fileName, localOnProgress, false, headers).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); diff --git a/src/loaders/splat/SplatLoader.js b/src/loaders/splat/SplatLoader.js index 59a535d..90e8009 100644 --- a/src/loaders/splat/SplatLoader.js +++ b/src/loaders/splat/SplatLoader.js @@ -23,7 +23,7 @@ function finalize(splatData, optimizeSplatData, minimumAlpha, compressionLevel, export class SplatLoader { static loadFromURL(fileName, onProgress, loadDirectoToSplatBuffer, onProgressiveLoadSectionProgress, minimumAlpha, compressionLevel, - optimizeSplatData = true, sectionSize, sceneCenter, blockSize, bucketSize) { + optimizeSplatData = true, headers, sectionSize, sceneCenter, blockSize, bucketSize) { let internalLoadType = loadDirectoToSplatBuffer ? InternalLoadType.DirectToSplatBuffer : InternalLoadType.DirectToSplatArray; if (optimizeSplatData) internalLoadType = InternalLoadType.DirectToSplatArray; @@ -149,7 +149,7 @@ export class SplatLoader { }; if (onProgress) onProgress(0, '0%', LoaderStatus.Downloading); - return fetchWithProgress(fileName, localOnProgress, false).then(() => { + return fetchWithProgress(fileName, localOnProgress, false, headers).then(() => { if (onProgress) onProgress(0, '0%', LoaderStatus.Processing); return loadPromise.promise.then((splatData) => { if (onProgress) onProgress(100, '100%', LoaderStatus.Done); diff --git a/src/splatmesh/SplatMaterial.js b/src/splatmesh/SplatMaterial.js index fde5c4a..584ea7e 100644 --- a/src/splatmesh/SplatMaterial.js +++ b/src/splatmesh/SplatMaterial.js @@ -140,7 +140,7 @@ export class SplatMaterial { if (dynamicMode) { vertexShaderSource += ` mat4 transform = transforms[sceneIndex]; - mat4 transformModelViewMatrix = modelViewMatrix * transform; + mat4 transformModelViewMatrix = viewMatrix * transform; `; } else { vertexShaderSource += `mat4 transformModelViewMatrix = modelViewMatrix;`; diff --git a/src/splatmesh/SplatMaterial3D.js b/src/splatmesh/SplatMaterial3D.js index ff4bdea..2f4ead3 100644 --- a/src/splatmesh/SplatMaterial3D.js +++ b/src/splatmesh/SplatMaterial3D.js @@ -17,8 +17,8 @@ export class SplatMaterial3D { * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats * @return {THREE.ShaderMaterial} */ - static build(dynamicMode = false, enableOptionalEffects = false, antialiased = false, - maxScreenSpaceSplatSize = 2048, splatScale = 1.0, pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0) { + static build(dynamicMode = false, enableOptionalEffects = false, antialiased = false, maxScreenSpaceSplatSize = 2048, + splatScale = 1.0, pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0, kernel2DSize = 0.3) { const customVertexVars = ` uniform vec2 covariancesTextureSize; @@ -38,7 +38,8 @@ export class SplatMaterial3D { let vertexShaderSource = SplatMaterial.buildVertexShaderBase(dynamicMode, enableOptionalEffects, maxSphericalHarmonicsDegree, customVertexVars); - vertexShaderSource += SplatMaterial3D.buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize); + vertexShaderSource += SplatMaterial3D.buildVertexShaderProjection(antialiased, enableOptionalEffects, + maxScreenSpaceSplatSize, kernel2DSize); const fragmentShaderSource = SplatMaterial3D.buildFragmentShader(); const uniforms = SplatMaterial.getUniforms(dynamicMode, enableOptionalEffects, @@ -76,7 +77,7 @@ export class SplatMaterial3D { return material; } - static buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize) { + static buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize, kernel2DSize) { let vertexShaderSource = ` vec4 sampledCovarianceA; @@ -136,16 +137,16 @@ export class SplatMaterial3D { if (antialiased) { vertexShaderSource += ` float detOrig = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; - cov2Dm[0][0] += 0.3; - cov2Dm[1][1] += 0.3; + cov2Dm[0][0] += ${kernel2DSize}; + cov2Dm[1][1] += ${kernel2DSize}; float detBlur = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1]; vColor.a *= sqrt(max(detOrig / detBlur, 0.0)); if (vColor.a < minAlpha) return; `; } else { vertexShaderSource += ` - cov2Dm[0][0] += 0.3; - cov2Dm[1][1] += 0.3; + cov2Dm[0][0] += ${kernel2DSize}; + cov2Dm[1][1] += ${kernel2DSize}; `; } diff --git a/src/splatmesh/SplatMesh.js b/src/splatmesh/SplatMesh.js index 6251efd..91ecbeb 100644 --- a/src/splatmesh/SplatMesh.js +++ b/src/splatmesh/SplatMesh.js @@ -49,7 +49,7 @@ export class SplatMesh extends THREE.Mesh { constructor(splatRenderMode = SplatRenderMode.ThreeD, dynamicMode = false, enableOptionalEffects = false, halfPrecisionCovariancesOnGPU = false, devicePixelRatio = 1, enableDistancesComputationOnGPU = true, integerBasedDistancesComputation = false, antialiased = false, maxScreenSpaceSplatSize = 1024, logLevel = LogLevel.None, - sphericalHarmonicsDegree = 0, sceneFadeInRateMultiplier = 1.0) { + sphericalHarmonicsDegree = 0, sceneFadeInRateMultiplier = 1.0, kernel2DSize = 0.3) { super(dummyGeometry, dummyMaterial); // Reference to a Three.js renderer @@ -88,6 +88,10 @@ export class SplatMesh extends THREE.Mesh { // https://github.com/graphdeco-inria/gaussian-splatting/issues/294#issuecomment-1772688093 this.antialiased = antialiased; + // The size of the 2D kernel used for splat rendering + // This will adjust the 2D kernel size after the projection + this.kernel2DSize = kernel2DSize; + // Specify the maximum clip space splat size, can help deal with large splats that get too unwieldy this.maxScreenSpaceSplatSize = maxScreenSpaceSplatSize; @@ -364,7 +368,7 @@ export class SplatMesh extends THREE.Mesh { if (this.splatRenderMode === SplatRenderMode.ThreeD) { this.material = SplatMaterial3D.build(this.dynamicMode, this.enableOptionalEffects, this.antialiased, this.maxScreenSpaceSplatSize, this.splatScale, this.pointCloudModeEnabled, - this.minSphericalHarmonicsDegree); + this.minSphericalHarmonicsDegree, this.kernel2DSize); } else { this.material = SplatMaterial2D.build(this.dynamicMode, this.enableOptionalEffects, this.splatScale, this.pointCloudModeEnabled, this.minSphericalHarmonicsDegree);