diff --git a/src/framework/parsers/ply.js b/src/framework/parsers/ply.js index 3ed2aa3ffc4..22a222aa969 100644 --- a/src/framework/parsers/ply.js +++ b/src/framework/parsers/ply.js @@ -198,18 +198,31 @@ const isCompressedPly = (elements) => { 'min_x', 'min_y', 'min_z', 'max_x', 'max_y', 'max_z', 'min_scale_x', 'min_scale_y', 'min_scale_z', - 'max_scale_x', 'max_scale_y', 'max_scale_z' + 'max_scale_x', 'max_scale_y', 'max_scale_z', + 'min_r', 'min_g', 'min_b', + 'max_r', 'max_g', 'max_b' ]; const vertexProperties = [ 'packed_position', 'packed_rotation', 'packed_scale', 'packed_color' ]; - return elements.length === 2 && - elements[0].name === 'chunk' && - elements[0].properties.every((p, i) => p.name === chunkProperties[i] && p.type === 'float') && - elements[1].name === 'vertex' && - elements[1].properties.every((p, i) => p.name === vertexProperties[i] && p.type === 'uint'); + const shProperties = new Array(45).fill('').map((_, i) => `f_rest_${i}`); + + const hasBaseElements = () => { + return elements[0].name === 'chunk' && + elements[0].properties.every((p, i) => p.name === chunkProperties[i] && p.type === 'float') && + elements[1].name === 'vertex' && + elements[1].properties.every((p, i) => p.name === vertexProperties[i] && p.type === 'uint'); + }; + + const hasSHElements = () => { + return elements[2].name === 'sh' && + [9, 24, 45].indexOf(elements[2].properties.length) !== -1 && + elements[2].properties.every((p, i) => p.name === shProperties[i] && p.type === 'uchar'); + }; + + return (elements.length === 2 && hasBaseElements()) || (elements.length === 3 && hasBaseElements() && hasSHElements()); }; const isFloatPly = (elements) => { @@ -223,10 +236,8 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { const result = new GSplatCompressedData(); const numChunks = elements[0].count; - const chunkSize = 12 * 4; - + const numChunkProperties = elements[0].properties.length; const numVertices = elements[1].count; - const vertexSize = 4 * 4; // evaluate the storage size for the given count (this must match the // texture size calculation in GSplatCompressed). @@ -237,64 +248,39 @@ const readCompressedPly = async (streamBuf, elements, littleEndian) => { }; // allocate result - result.numSplats = elements[1].count; - result.chunkData = new Float32Array(evalStorageSize(numChunks) * 12); + result.numSplats = numVertices; + result.chunkData = new Float32Array(numChunks * numChunkProperties); result.vertexData = new Uint32Array(evalStorageSize(numVertices) * 4); - let uint32StreamData; - const uint32ChunkData = new Uint32Array(result.chunkData.buffer); - const uint32VertexData = result.vertexData; - - // read chunks - let chunks = 0; - while (chunks < numChunks) { - while (streamBuf.remaining < chunkSize) { - /* eslint-disable no-await-in-loop */ - await streamBuf.read(); - } - - // ensure the uint32 view is still valid - if (uint32StreamData?.buffer !== streamBuf.data.buffer) { - uint32StreamData = new Uint32Array(streamBuf.data.buffer, 0, Math.floor(streamBuf.data.buffer.byteLength / 4)); - } - - // read the next chunk of data - const toRead = Math.min(numChunks - chunks, Math.floor(streamBuf.remaining / chunkSize)); + // read length bytes of data into buffer + const read = async (buffer, length) => { + const target = new Uint8Array(buffer); + let cursor = 0; - const dstOffset = chunks * 12; - const srcOffset = streamBuf.head / 4; - for (let i = 0; i < toRead * 12; ++i) { - uint32ChunkData[dstOffset + i] = uint32StreamData[srcOffset + i]; - } - - streamBuf.head += toRead * chunkSize; - chunks += toRead; - } - - // read vertices - let vertices = 0; - while (vertices < numVertices) { - while (streamBuf.remaining < vertexSize) { - /* eslint-disable no-await-in-loop */ - await streamBuf.read(); - } + while (cursor < length) { + while (streamBuf.remaining === 0) { + /* eslint-disable no-await-in-loop */ + await streamBuf.read(); + } - // ensure the uint32 view is still valid - if (uint32StreamData?.buffer !== streamBuf.data.buffer) { - uint32StreamData = new Uint32Array(streamBuf.data.buffer, 0, Math.floor(streamBuf.data.buffer.byteLength / 4)); + const toCopy = Math.min(length - cursor, streamBuf.remaining); + const src = streamBuf.data; + for (let i = 0; i < toCopy; ++i) { + target[cursor++] = src[streamBuf.head++]; + } } + }; - // read the next chunk of data - const toRead = Math.min(numVertices - vertices, Math.floor(streamBuf.remaining / vertexSize)); + // read chunk data + await read(result.chunkData.buffer, numChunks * numChunkProperties * 4); - const dstOffset = vertices * 4; - const srcOffset = streamBuf.head / 4; - for (let i = 0; i < toRead * 4; ++i) { - uint32VertexData[dstOffset + i] = uint32StreamData[srcOffset + i]; - } + // read packed vertices + await read(result.vertexData.buffer, numVertices * 4 * 4); - streamBuf.head += toRead * vertexSize; - vertices += toRead; + // read sh data + if (elements.length === 3) { + result.shData = new Uint8Array(elements[2].count * elements[2].properties.length); + await read(result.shData.buffer, result.shData.byteLength); } return result; diff --git a/src/platform/graphics/texture.js b/src/platform/graphics/texture.js index b630d1e9077..af2d59ed8df 100644 --- a/src/platform/graphics/texture.js +++ b/src/platform/graphics/texture.js @@ -163,7 +163,7 @@ class Texture { * - {@link FUNC_NOTEQUAL} * * Defaults to {@link FUNC_LESS}. - * @param {Uint8Array[]|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] + * @param {Uint8Array[]|Uint16Array[]|Uint32Array[]|Float32Array[]|HTMLCanvasElement[]|HTMLImageElement[]|HTMLVideoElement[]|Uint8Array[][]} [options.levels] * - Array of Uint8Array or other supported browser interface; or a two-dimensional array * of Uint8Array if options.arrayLength is defined and greater than zero. * @param {boolean} [options.storage] - Defines if texture can be used as a storage texture by diff --git a/src/scene/gsplat/gsplat-compressed-data.js b/src/scene/gsplat/gsplat-compressed-data.js index b938b01e27e..28f93978a57 100644 --- a/src/scene/gsplat/gsplat-compressed-data.js +++ b/src/scene/gsplat/gsplat-compressed-data.js @@ -7,7 +7,7 @@ const SH_C0 = 0.28209479177387814; // iterator for accessing compressed splat data class SplatCompressedIterator { - constructor(gsplatData, p, r, s, c) { + constructor(gsplatData, p, r, s, c, sh) { const unpackUnorm = (value, bits) => { const t = (1 << bits) - 1; return (value & t) / t; @@ -47,11 +47,11 @@ class SplatCompressedIterator { return (a === b) ? a : a * (1 - t) + b * t; }; - const chunkData = gsplatData.chunkData; - const vertexData = gsplatData.vertexData; + const { chunkData, chunkSize, vertexData, shData, shBands } = gsplatData; + const shCoeffs = [3, 8, 15][shBands - 1]; this.read = (i) => { - const ci = Math.floor(i / 256) * 12; + const ci = Math.floor(i / 256) * chunkSize; if (p) { unpack111011(p, vertexData[i * 4 + 0]); @@ -73,6 +73,19 @@ class SplatCompressedIterator { if (c) { unpack8888(c, vertexData[i * 4 + 3]); + if (chunkSize > 12) { + c.x = lerp(chunkData[ci + 12], chunkData[ci + 15], c.x); + c.y = lerp(chunkData[ci + 13], chunkData[ci + 16], c.y); + c.z = lerp(chunkData[ci + 14], chunkData[ci + 17], c.z); + } + } + + if (sh && shBands > 0) { + for (let j = 0; j < 3; ++j) { + for (let k = 0; k < 15; ++k) { + sh[j * 15 + k] = (k < shCoeffs) ? (shData[(i * 3 + j) * shCoeffs + k] * (8 / 255) - 4) : 0; + } + } } }; } @@ -82,11 +95,13 @@ class GSplatCompressedData { numSplats; /** - * Contains 12 floats per chunk: + * Contains either 12 or 18 floats per chunk: * min_x, min_y, min_z, * max_x, max_y, max_z, * min_scale_x, min_scale_y, min_scale_z, * max_scale_x, max_scale_y, max_scale_z + * min_r, min_g, min_b, + * max_r, max_g, max_b * @type {Float32Array} */ chunkData; @@ -101,6 +116,12 @@ class GSplatCompressedData { */ vertexData; + /** + * Contains optional quantized spherical harmonic data for up to 3 bands. + * @type {Uint8Array} + */ + shData; + /** * Create an iterator for accessing splat data * @@ -108,10 +129,11 @@ class GSplatCompressedData { * @param {Quat|null} [r] - the quaternion to receive splat rotation * @param {Vec3|null} [s] - the vector to receive splat scale * @param {Vec4|null} [c] - the vector to receive splat color + * @param {Float32Array|null} [sh] - the array to receive spherical harmonics data * @returns {SplatCompressedIterator} - The iterator */ - createIter(p, r, s, c) { - return new SplatCompressedIterator(this, p, r, s, c); + createIter(p, r, s, c, sh) { + return new SplatCompressedIterator(this, p, r, s, c, sh); } /** @@ -122,29 +144,25 @@ class GSplatCompressedData { * @returns {boolean} - Whether the calculation was successful. */ calcAabb(result) { - let mx, my, mz, Mx, My, Mz; - - // fast bounds calc using chunk data - const numChunks = Math.ceil(this.numSplats / 256); - - const chunkData = this.chunkData; + const { chunkData, numChunks, chunkSize } = this; let s = Math.exp(Math.max(chunkData[9], chunkData[10], chunkData[11])); - mx = chunkData[0] - s; - my = chunkData[1] - s; - mz = chunkData[2] - s; - Mx = chunkData[3] + s; - My = chunkData[4] + s; - Mz = chunkData[5] + s; + let mx = chunkData[0] - s; + let my = chunkData[1] - s; + let mz = chunkData[2] - s; + let Mx = chunkData[3] + s; + let My = chunkData[4] + s; + let Mz = chunkData[5] + s; for (let i = 1; i < numChunks; ++i) { - s = Math.exp(Math.max(chunkData[i * 12 + 9], chunkData[i * 12 + 10], chunkData[i * 12 + 11])); - mx = Math.min(mx, chunkData[i * 12 + 0] - s); - my = Math.min(my, chunkData[i * 12 + 1] - s); - mz = Math.min(mz, chunkData[i * 12 + 2] - s); - Mx = Math.max(Mx, chunkData[i * 12 + 3] + s); - My = Math.max(My, chunkData[i * 12 + 4] + s); - Mz = Math.max(Mz, chunkData[i * 12 + 5] + s); + const off = i * chunkSize; + s = Math.exp(Math.max(chunkData[off + 9], chunkData[off + 10], chunkData[off + 11])); + mx = Math.min(mx, chunkData[off + 0] - s); + my = Math.min(my, chunkData[off + 1] - s); + mz = Math.min(mz, chunkData[off + 2] - s); + Mx = Math.max(Mx, chunkData[off + 3] + s); + My = Math.max(My, chunkData[off + 4] + s); + Mz = Math.max(Mz, chunkData[off + 5] + s); } result.center.set((mx + Mx) * 0.5, (my + My) * 0.5, (mz + Mz) * 0.5); @@ -157,20 +175,18 @@ class GSplatCompressedData { * @param {Float32Array} result - Array containing the centers. */ getCenters(result) { - const chunkData = this.chunkData; - const vertexData = this.vertexData; - - const numChunks = Math.ceil(this.numSplats / 256); + const { vertexData, chunkData, numChunks, chunkSize } = this; let mx, my, mz, Mx, My, Mz; for (let c = 0; c < numChunks; ++c) { - mx = chunkData[c * 12 + 0]; - my = chunkData[c * 12 + 1]; - mz = chunkData[c * 12 + 2]; - Mx = chunkData[c * 12 + 3]; - My = chunkData[c * 12 + 4]; - Mz = chunkData[c * 12 + 5]; + const off = c * chunkSize; + mx = chunkData[off + 0]; + my = chunkData[off + 1]; + mz = chunkData[off + 2]; + Mx = chunkData[off + 3]; + My = chunkData[off + 4]; + Mz = chunkData[off + 5]; const end = Math.min(this.numSplats, (c + 1) * 256); for (let i = c * 256; i < end; ++i) { @@ -189,17 +205,17 @@ class GSplatCompressedData { * @param {Vec3} result - The result. */ calcFocalPoint(result) { - const chunkData = this.chunkData; - const numChunks = Math.ceil(this.numSplats / 256); + const { chunkData, numChunks, chunkSize } = this; result.x = 0; result.y = 0; result.z = 0; for (let i = 0; i < numChunks; ++i) { - result.x += chunkData[i * 12 + 0] + chunkData[i * 12 + 3]; - result.y += chunkData[i * 12 + 1] + chunkData[i * 12 + 4]; - result.z += chunkData[i * 12 + 2] + chunkData[i * 12 + 5]; + const off = i * chunkSize; + result.x += chunkData[off + 0] + chunkData[off + 3]; + result.y += chunkData[off + 1] + chunkData[off + 4]; + result.z += chunkData[off + 2] + chunkData[off + 5]; } result.mulScalar(0.5 / numChunks); } @@ -208,10 +224,38 @@ class GSplatCompressedData { return true; } + get numChunks() { + return Math.ceil(this.numSplats / 256); + } + + get chunkSize() { + return this.chunkData.length / this.numChunks; + } + + get shBands() { + const sizes = { + 3: 1, + 8: 2, + 15: 3 + }; + return sizes[this.shData?.length / this.numSplats / 3] ?? 0; + } + // decompress into GSplatData decompress() { const members = ['x', 'y', 'z', 'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity', 'rot_0', 'rot_1', 'rot_2', 'rot_3', 'scale_0', 'scale_1', 'scale_2']; + const { shBands } = this; + + // allocate spherical harmonics data + if (shBands > 0) { + const shMembers = []; + for (let i = 0; i < 45; ++i) { + shMembers.push(`f_rest_${i}`); + } + members.splice(members.indexOf('f_dc_0') + 1, 0, ...shMembers); + } + // allocate uncompressed data const data = {}; members.forEach((name) => { @@ -222,8 +266,9 @@ class GSplatCompressedData { const r = new Quat(); const s = new Vec3(); const c = new Vec4(); + const sh = shBands > 0 ? new Float32Array(45) : null; - const iter = this.createIter(p, r, s, c); + const iter = this.createIter(p, r, s, c, sh); for (let i = 0; i < this.numSplats; ++i) { iter.read(i); @@ -246,6 +291,12 @@ class GSplatCompressedData { data.f_dc_2[i] = (c.z - 0.5) / SH_C0; // convert opacity to log sigmoid taking into account infinities at 0 and 1 data.opacity[i] = (c.w <= 0) ? -40 : (c.w >= 1) ? 40 : -Math.log(1 / c.w - 1); + + if (sh) { + for (let c = 0; c < 45; ++c) { + data[`f_rest_${c}`][i] = sh[c]; + } + } } return new GSplatData([{ diff --git a/src/scene/gsplat/gsplat-compressed-material.js b/src/scene/gsplat/gsplat-compressed-material.js index 31281c218da..f8d22754bf0 100644 --- a/src/scene/gsplat/gsplat-compressed-material.js +++ b/src/scene/gsplat/gsplat-compressed-material.js @@ -37,6 +37,8 @@ const splatCoreVS = /* glsl */ ` vec4 chunkDataA; // x: min_x, y: min_y, z: min_z, w: max_x vec4 chunkDataB; // x: max_y, y: max_z, z: scale_min_x, w: scale_min_y vec4 chunkDataC; // x: scale_min_z, y: scale_max_x, z: scale_max_y, w: scale_max_z + vec4 chunkDataD; // x: min_r, y: min_g, z: min_b, w: max_r + vec4 chunkDataE; // x: max_g, y: max_b, z: unused, w: unused uvec4 packedData; // x: position bits, y: rotation bits, z: scale bits, w: color bits // calculate the current splat index and uvs @@ -67,7 +69,7 @@ const splatCoreVS = /* glsl */ ` // calculate chunkUV uint chunkId = splatId / 256u; chunkUV = ivec2( - int((chunkId % chunkWidth) * 3u), + int((chunkId % chunkWidth) * 5u), int(chunkId / chunkWidth) ); @@ -79,6 +81,8 @@ const splatCoreVS = /* glsl */ ` chunkDataA = texelFetch(chunkTexture, chunkUV, 0); chunkDataB = texelFetch(chunkTexture, ivec2(chunkUV.x + 1, chunkUV.y), 0); chunkDataC = texelFetch(chunkTexture, ivec2(chunkUV.x + 2, chunkUV.y), 0); + chunkDataD = texelFetch(chunkTexture, ivec2(chunkUV.x + 3, chunkUV.y), 0); + chunkDataE = texelFetch(chunkTexture, ivec2(chunkUV.x + 4, chunkUV.y), 0); packedData = texelFetch(packedTexture, packedUV, 0); } @@ -127,7 +131,8 @@ const splatCoreVS = /* glsl */ ` } vec4 getColor() { - return unpack8888(packedData.w); + vec4 r = unpack8888(packedData.w); + return vec4(mix(chunkDataD.xyz, vec3(chunkDataD.w, chunkDataE.xy), r.rgb), r.w); } mat3 quatToMat3(vec4 R) { @@ -200,6 +205,117 @@ const splatCoreVS = /* glsl */ ` return vec4(v1, v2); } + +#if defined(USE_SH) + #define SH_C1 0.4886025119029199f + + #define SH_C2_0 1.0925484305920792f + #define SH_C2_1 -1.0925484305920792f + #define SH_C2_2 0.31539156525252005f + #define SH_C2_3 -1.0925484305920792f + #define SH_C2_4 0.5462742152960396f + + #define SH_C3_0 -0.5900435899266435f + #define SH_C3_1 2.890611442640554f + #define SH_C3_2 -0.4570457994644658f + #define SH_C3_3 0.3731763325901154f + #define SH_C3_4 -0.4570457994644658f + #define SH_C3_5 1.445305721320277f + #define SH_C3_6 -0.5900435899266435f + + uniform highp usampler2D shTexture0; + uniform highp usampler2D shTexture1; + uniform highp usampler2D shTexture2; + + vec4 sunpack8888(in uint bits) { + return vec4((uvec4(bits) >> uvec4(0u, 8u, 16u, 24u)) & 0xffu) * (8.0 / 255.0) - 4.0; + } + + void readSHData(out vec3 sh[15]) { + // read the sh coefficients + uvec4 shData0 = texelFetch(shTexture0, packedUV, 0); + uvec4 shData1 = texelFetch(shTexture1, packedUV, 0); + uvec4 shData2 = texelFetch(shTexture2, packedUV, 0); + + vec4 r0 = sunpack8888(shData0.x); + vec4 r1 = sunpack8888(shData0.y); + vec4 r2 = sunpack8888(shData0.z); + vec4 r3 = sunpack8888(shData0.w); + + vec4 g0 = sunpack8888(shData1.x); + vec4 g1 = sunpack8888(shData1.y); + vec4 g2 = sunpack8888(shData1.z); + vec4 g3 = sunpack8888(shData1.w); + + vec4 b0 = sunpack8888(shData2.x); + vec4 b1 = sunpack8888(shData2.y); + vec4 b2 = sunpack8888(shData2.z); + vec4 b3 = sunpack8888(shData2.w); + + sh[0] = vec3(r0.x, g0.x, b0.x); + sh[1] = vec3(r0.y, g0.y, b0.y); + sh[2] = vec3(r0.z, g0.z, b0.z); + sh[3] = vec3(r0.w, g0.w, b0.w); + sh[4] = vec3(r1.x, g1.x, b1.x); + sh[5] = vec3(r1.y, g1.y, b1.y); + sh[6] = vec3(r1.z, g1.z, b1.z); + sh[7] = vec3(r1.w, g1.w, b1.w); + sh[8] = vec3(r2.x, g2.x, b2.x); + sh[9] = vec3(r2.y, g2.y, b2.y); + sh[10] = vec3(r2.z, g2.z, b2.z); + sh[11] = vec3(r2.w, g2.w, b2.w); + sh[12] = vec3(r3.x, g3.x, b3.x); + sh[13] = vec3(r3.y, g3.y, b3.y); + sh[14] = vec3(r3.z, g3.z, b3.z); + } + + // see https://github.com/graphdeco-inria/gaussian-splatting/blob/main/utils/sh_utils.py + vec3 evalSH(in vec3 dir) { + + vec3 sh[15]; + readSHData(sh); + + vec3 result = vec3(0.0); + + // 1st degree + float x = dir.x; + float y = dir.y; + float z = dir.z; + + result += SH_C1 * (-sh[0] * y + sh[1] * z - sh[2] * x); + + // 2nd degree + float xx = x * x; + float yy = y * y; + float zz = z * z; + float xy = x * y; + float yz = y * z; + float xz = x * z; + + result += + sh[3] * (SH_C2_0 * xy) * + + sh[4] * (SH_C2_1 * yz) + + sh[5] * (SH_C2_2 * (2.0 * zz - xx - yy)) + + sh[6] * (SH_C2_3 * xz) + + sh[7] * (SH_C2_4 * (xx - yy)); + + // 3rd degree + result += + sh[8] * (SH_C3_0 * y * (3.0 * xx - yy)) + + sh[9] * (SH_C3_1 * xy * z) + + sh[10] * (SH_C3_2 * y * (4.0 * zz - xx - yy)) + + sh[11] * (SH_C3_3 * z * (2.0 * zz - 3.0 * xx - 3.0 * yy)) + + sh[12] * (SH_C3_4 * x * (4.0 * zz - xx - yy)) + + sh[13] * (SH_C3_5 * z * (xx - yy)) + + sh[14] * (SH_C3_6 * x * (xx - 3.0 * yy)); + + return result; + } +#else + vec3 evalSH(in vec3 dir) { + return vec3(0.0); + } +#endif `; const splatCoreFS = /* glsl */ ` @@ -252,10 +368,12 @@ class GSplatCompressedShaderGenerator { const shaderPassInfo = ShaderPass.get(device).getByIndex(options.pass); const shaderPassDefines = shaderPassInfo.shaderDefines; + const optionDefines = (options.defines ?? []).map(d => `#define ${d}`).join('\n'); const defines = - `${shaderPassDefines - }#define DITHER_${options.dither.toUpperCase()}\n` + + `${shaderPassDefines}\n` + + `${optionDefines}\n` + + `#define DITHER_${options.dither.toUpperCase()}\n` + `#define TONEMAP_${options.toneMapping === TONEMAP_LINEAR ? 'DISABLED' : 'ENABLED'}\n`; const vs = defines + splatCoreVS + options.vertex; @@ -283,6 +401,8 @@ const splatMainVS = /* glsl */ ` varying mediump vec2 texCoord; varying mediump vec4 color; + uniform vec3 view_position; + mediump vec4 discardVec = vec4(0.0, 0.0, 2.0, 1.0); void main(void) @@ -334,6 +454,12 @@ const splatMainVS = /* glsl */ ` texCoord = vertex_position.xy * scale / 2.0; + #ifdef USE_SH + vec4 worldCenter = matrix_model * vec4(center, 1.0); + vec3 viewDir = normalize((worldCenter.xyz / worldCenter.w - view_position) * mat3(matrix_model)); + color.xyz = max(color.xyz + evalSH(viewDir), 0.0); + #endif + #ifndef DITHER_NONE id = float(splatId); #endif @@ -355,6 +481,7 @@ const splatMainFS = /* glsl */ ` * @property {string} [vertex] - Custom vertex shader, see SPLAT MANY example. * @property {string} [fragment] - Custom fragment shader, see SPLAT MANY example. * @property {string} [dither] - Opacity dithering enum. + * @property {string[]} [defines] - List of shader defines. */ /** @@ -380,7 +507,8 @@ const createGSplatCompressedMaterial = (options = {}) => { toneMapping: (pass === SHADER_FORWARDHDR ? TONEMAP_LINEAR : scene.toneMapping), vertex: options.vertex ?? splatMainVS, fragment: options.fragment ?? splatMainFS, - dither: ditherEnum + dither: ditherEnum, + defines: options.defines }; const processingOptions = new ShaderProcessorOptions(viewUniformFormat, viewBindGroupFormat); diff --git a/src/scene/gsplat/gsplat-compressed.js b/src/scene/gsplat/gsplat-compressed.js index a545f0eed24..6bc14ba3754 100644 --- a/src/scene/gsplat/gsplat-compressed.js +++ b/src/scene/gsplat/gsplat-compressed.js @@ -6,6 +6,22 @@ import { Texture } from '../../platform/graphics/texture.js'; import { BoundingBox } from '../../core/shape/bounding-box.js'; import { createGSplatCompressedMaterial } from './gsplat-compressed-material.js'; +/** + * @import { GSplatCompressedData } from './gsplat-compressed-data.js' + * @import { GraphicsDevice } from '../../platform/graphics/graphics-device.js' + * @import { Material } from '../materials/material.js' + * @import { SplatMaterialOptions } from './gsplat-material.js' + */ + +// copy data with padding +const strideCopy = (target, targetStride, src, srcStride, numEntries) => { + for (let i = 0; i < numEntries; ++i) { + for (let j = 0; j < srcStride; ++j) { + target[i * targetStride + j] = src[i * srcStride + j]; + } + } +}; + /** @ignore */ class GSplatCompressed { device; @@ -24,13 +40,21 @@ class GSplatCompressed { /** @type {Texture} */ chunkTexture; + /** @type {Texture?} */ + shTexture0; + + /** @type {Texture?} */ + shTexture1; + + /** @type {Texture?} */ + shTexture2; + /** * @param {import('../../platform/graphics/graphics-device.js').GraphicsDevice} device - The graphics device. * @param {import('./gsplat-compressed-data.js').GSplatCompressedData} gsplatData - The splat data. */ constructor(device, gsplatData) { - const numSplats = gsplatData.numSplats; - const numChunks = Math.ceil(numSplats / 256); + const { chunkData, chunkSize, numChunks, numSplats, vertexData, shBands } = gsplatData; this.device = device; this.numSplats = numSplats; @@ -40,22 +64,79 @@ class GSplatCompressed { gsplatData.calcAabb(this.aabb); // initialize centers - this.centers = new Float32Array(gsplatData.numSplats * 3); + this.centers = new Float32Array(numSplats * 3); gsplatData.getCenters(this.centers); // initialize packed data - this.packedTexture = this.createTexture('packedData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), gsplatData.vertexData); + this.packedTexture = this.createTexture('packedData', PIXELFORMAT_RGBA32U, this.evalTextureSize(numSplats), vertexData); // initialize chunk data - const chunkSize = this.evalTextureSize(numChunks); - chunkSize.x *= 3; - - this.chunkTexture = this.createTexture('chunkData', PIXELFORMAT_RGBA32F, chunkSize, gsplatData.chunkData); + const chunkTextureSize = this.evalTextureSize(numChunks); + chunkTextureSize.x *= 5; + + this.chunkTexture = this.createTexture('chunkData', PIXELFORMAT_RGBA32F, chunkTextureSize); + const chunkTextureData = this.chunkTexture.lock(); + strideCopy(chunkTextureData, 20, chunkData, chunkSize, numChunks); + + if (chunkSize === 12) { + // if the chunks don't contain color min/max values we must update max to 1 (min is filled with 0's) + for (let i = 0; i < numChunks; ++i) { + chunkTextureData[i * 20 + 15] = 1; + chunkTextureData[i * 20 + 16] = 1; + chunkTextureData[i * 20 + 17] = 1; + } + } + + this.chunkTexture.unlock(); + + // load optional spherical harmonics data + if (shBands > 0) { + const { shData } = gsplatData; + + const size = this.evalTextureSize(numSplats); + + const texture0 = this.createTexture('shTexture0', PIXELFORMAT_RGBA32U, size); + const texture1 = this.createTexture('shTexture1', PIXELFORMAT_RGBA32U, size); + const texture2 = this.createTexture('shTexture2', PIXELFORMAT_RGBA32U, size); + + const data0 = texture0.lock(); + const data1 = texture1.lock(); + const data2 = texture2.lock(); + + const target0 = new Uint8Array(data0.buffer); + const target1 = new Uint8Array(data1.buffer); + const target2 = new Uint8Array(data2.buffer); + + const srcCoeffs = [3, 8, 15][shBands - 1]; + + for (let i = 0; i < numSplats; ++i) { + for (let j = 0; j < srcCoeffs; ++j) { + target0[i * 16 + j] = shData[(i * 3 + 0) * srcCoeffs + j]; + target1[i * 16 + j] = shData[(i * 3 + 1) * srcCoeffs + j]; + target2[i * 16 + j] = shData[(i * 3 + 2) * srcCoeffs + j]; + } + } + + texture0.unlock(); + texture1.unlock(); + texture2.unlock(); + + this.shTexture0 = texture0; + this.shTexture1 = texture1; + this.shTexture2 = texture2; + } else { + this.shTexture0 = null; + this.shTexture1 = null; + this.shTexture2 = null; + } } destroy() { this.packedTexture?.destroy(); this.chunkTexture?.destroy(); + this.shTexture0?.destroy(); + this.shTexture1?.destroy(); + this.shTexture2?.destroy(); } /** @@ -63,10 +144,19 @@ class GSplatCompressed { * the splat rendering. */ createMaterial(options) { - const result = createGSplatCompressedMaterial(options); + const hasSH = this.shTexture0 !== null; + const result = createGSplatCompressedMaterial({ + ...(hasSH ? { defines: ['USE_SH'] } : { }), + ...options + }); result.setParameter('packedTexture', this.packedTexture); result.setParameter('chunkTexture', this.chunkTexture); - result.setParameter('tex_params', new Float32Array([this.numSplats, this.packedTexture.width, this.chunkTexture.width / 3, 0])); + result.setParameter('tex_params', new Float32Array([this.numSplats, this.packedTexture.width, this.chunkTexture.width / 5, 0])); + if (hasSH) { + result.setParameter('shTexture0', this.shTexture0); + result.setParameter('shTexture1', this.shTexture1); + result.setParameter('shTexture2', this.shTexture2); + } return result; } @@ -90,6 +180,7 @@ class GSplatCompressed { * @param {string} name - The name of the texture to be created. * @param {number} format - The pixel format of the texture. * @param {Vec2} size - The width and height of the texture. + * @param {Uint8Array|Uint16Array|Uint32Array} [data] - The initial data to fill the texture with. * @returns {Texture} The created texture instance. */ createTexture(name, format, size, data) { diff --git a/src/scene/gsplat/gsplat-data.js b/src/scene/gsplat/gsplat-data.js index d44df1ba64f..1e6171afa9a 100644 --- a/src/scene/gsplat/gsplat-data.js +++ b/src/scene/gsplat/gsplat-data.js @@ -316,6 +316,15 @@ class GSplatData { return false; } + get hasSHData() { + for (let i = 0; i < 45; ++i) { + if (!this.getProp(`f_rest_${i}`)) { + return false; + } + } + return true; + } + calcMortonOrder() { const calcMinMax = (arr) => { let min = arr[0]; diff --git a/src/scene/gsplat/gsplat.js b/src/scene/gsplat/gsplat.js index 39254fc2d03..fdb34521a7a 100644 --- a/src/scene/gsplat/gsplat.js +++ b/src/scene/gsplat/gsplat.js @@ -81,7 +81,7 @@ class GSplat { this.updateTransformData(gsplatData); // initialize SH data - this.hasSH = getSHData(gsplatData).every(x => x); + this.hasSH = gsplatData.hasSHData; if (this.hasSH) { this.sh1to3Texture = this.createTexture('splatSH_1to3', PIXELFORMAT_RGBA32U, size); this.sh4to7Texture = this.createTexture('splatSH_4to7', PIXELFORMAT_RGBA32U, size);