Skip to content

feat(SPLAT): Replace manual SPZ loader with NianticLabs WASM implementation and enable 4-th order spherical harmonics #18267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
raymondyfei wants to merge 9 commits into
base: master
Choose a base branch
from
Open

feat(SPLAT): Replace manual SPZ loader with NianticLabs WASM implementation and enable 4-th order spherical harmonics #18267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a06fb4c
replace the SPZ loader with the WASM package
Apr 7, 2026
69be098
perf opt
Apr 7, 2026
7914101
bump SPZ version
Apr 9, 2026
42a01ec
Merge branch 'master' into yfei/spz_wasm_loader
Apr 9, 2026
32aff3a
bump SPZ version
Apr 9, 2026
1f49393
fix endings
Apr 9, 2026
0d6451c
Add 4th-order SH loading and rendering
Apr 10, 2026
37d11da
minor
Apr 10, 2026
f431b32
add unit tests for SPZ and 4-th order SHs
Apr 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 13 additions & 2 deletions packages/dev/core/src/Materials/GaussianSplatting/gaussianSplattingMaterial.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,18 @@ export class GaussianSplattingMaterial extends PushMaterial {
}

protected static _Attribs = [VertexBuffer.PositionKind, "splatIndex0", "splatIndex1", "splatIndex2", "splatIndex3"];
protected static _Samplers = ["covariancesATexture", "covariancesBTexture", "centersTexture", "colorsTexture", "shTexture0", "shTexture1", "shTexture2", "partIndicesTexture"];
protected static _Samplers = [
"covariancesATexture",
"covariancesBTexture",
"centersTexture",
"colorsTexture",
"shTexture0",
"shTexture1",
"shTexture2",
"shTexture3",
"shTexture4",
"partIndicesTexture",
];
protected static _UniformBuffers = ["Scene", "Mesh"];
protected static _Uniforms = [
"world",
Expand Down Expand Up @@ -435,7 +446,7 @@ export class GaussianSplattingMaterial extends PushMaterial {
effect.setTexture("colorsTexture", gsMesh.colorsTexture);

if (gsMesh.shTextures) {
for (let i = 0; i < gsMesh.shTextures?.length; i++) {
for (let i = 0; i < gsMesh.shTextures.length; i++) {
effect.setTexture(`shTexture${i}`, gsMesh.shTextures[i]);
}
}
Expand Down
119 changes: 103 additions & 16 deletions packages/dev/core/src/Meshes/GaussianSplatting/gaussianSplattingMeshBase.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,33 @@ const enum PLYValue {
SH_42,
SH_43,
SH_44,
SH_45,
SH_46,
SH_47,
SH_48,
SH_49,
SH_50,
SH_51,
SH_52,
SH_53,
SH_54,
SH_55,
SH_56,
SH_57,
SH_58,
SH_59,
SH_60,
SH_61,
SH_62,
SH_63,
SH_64,
SH_65,
SH_66,
SH_67,
SH_68,
SH_69,
SH_70,
SH_71,

UNDEFINED,
}
Expand Down Expand Up @@ -344,6 +371,7 @@ export class GaussianSplattingMeshBase extends Mesh {
private static _PlyConversionBatchSize = 32768;
/** @internal */
public _shDegree = 0;
private _maxShDegree = 0;

private static readonly _BatchSize = 16; // 16 splats per instance
private _cameraViewInfos = new Map<number, ICameraViewInfo>();
Expand Down Expand Up @@ -391,7 +419,7 @@ export class GaussianSplattingMeshBase extends Mesh {
}

public set shDegree(value: number) {
const maxDegree = this._shTextures?.length ?? 0;
const maxDegree = this._maxShDegree;
const clamped = Math.max(0, Math.min(Math.round(value), maxDegree));
if (this._shDegree === clamped) {
return;
Expand All @@ -404,7 +432,7 @@ export class GaussianSplattingMeshBase extends Mesh {
* Maximum SH degree available from the loaded data.
*/
public get maxShDegree() {
return this._shTextures?.length ?? 0;
return this._maxShDegree;
}

/**
Expand Down Expand Up @@ -978,6 +1006,60 @@ export class GaussianSplattingMeshBase extends Mesh {
return PLYValue.SH_43;
case "f_rest_44":
return PLYValue.SH_44;
case "f_rest_45":
return PLYValue.SH_45;
case "f_rest_46":
return PLYValue.SH_46;
case "f_rest_47":
return PLYValue.SH_47;
case "f_rest_48":
return PLYValue.SH_48;
case "f_rest_49":
return PLYValue.SH_49;
case "f_rest_50":
return PLYValue.SH_50;
case "f_rest_51":
return PLYValue.SH_51;
case "f_rest_52":
return PLYValue.SH_52;
case "f_rest_53":
return PLYValue.SH_53;
case "f_rest_54":
return PLYValue.SH_54;
case "f_rest_55":
return PLYValue.SH_55;
case "f_rest_56":
return PLYValue.SH_56;
case "f_rest_57":
return PLYValue.SH_57;
case "f_rest_58":
return PLYValue.SH_58;
case "f_rest_59":
return PLYValue.SH_59;
case "f_rest_60":
return PLYValue.SH_60;
case "f_rest_61":
return PLYValue.SH_61;
case "f_rest_62":
return PLYValue.SH_62;
case "f_rest_63":
return PLYValue.SH_63;
case "f_rest_64":
return PLYValue.SH_64;
case "f_rest_65":
return PLYValue.SH_65;
case "f_rest_66":
return PLYValue.SH_66;
case "f_rest_67":
return PLYValue.SH_67;
case "f_rest_68":
return PLYValue.SH_68;
case "f_rest_69":
return PLYValue.SH_69;
case "f_rest_70":
return PLYValue.SH_70;
case "f_rest_71":
return PLYValue.SH_71;
}

return PLYValue.UNDEFINED;
Expand Down Expand Up @@ -1032,10 +1114,12 @@ export class GaussianSplattingMeshBase extends Mesh {

const value = GaussianSplattingMeshBase._ValueNameToEnum(name);
if (value != PLYValue.UNDEFINED) {
// SH degree 1,2 or 3 for 9, 24 or 45 values
if (value >= PLYValue.SH_44) {
shDegree = 3;
} else if (value >= PLYValue.SH_24) {
// SH degree 1,2,3 or 4 for 9, 24, 45 or 72 values
if (value >= PLYValue.SH_71) {
shDegree = 4;
} else if (value >= PLYValue.SH_44) {
shDegree = Math.max(shDegree, 3);
} else if (value >= PLYValue.SH_23) {
shDegree = Math.max(shDegree, 2);
} else if (value >= PLYValue.SH_8) {
shDegree = Math.max(shDegree, 1);
Expand Down Expand Up @@ -1323,7 +1407,7 @@ export class GaussianSplattingMeshBase extends Mesh {
r3 = value;
break;
}
if (sh && property.value >= PLYValue.SH_0 && property.value <= PLYValue.SH_44) {
if (sh && property.value >= PLYValue.SH_0 && property.value <= PLYValue.SH_71) {
const shIndex = property.value - PLYValue.SH_0;
if (property.type == PLYType.UCHAR && header.chunkCount) {
// compressed ply. dataView points to beginning of vertex
Expand All @@ -1341,7 +1425,7 @@ export class GaussianSplattingMeshBase extends Mesh {
}

if (sh) {
const shDim = header.shDegree == 1 ? 3 : header.shDegree == 2 ? 8 : 15;
const shDim = header.shDegree == 1 ? 3 : header.shDegree == 2 ? 8 : header.shDegree == 3 ? 15 : 24;
for (let j = 0; j < shDim; j++) {
sh[j * 3 + 0] = plySH[j];
sh[j * 3 + 1] = plySH[j + shDim];
Expand Down Expand Up @@ -1417,7 +1501,7 @@ export class GaussianSplattingMeshBase extends Mesh {
}
}

return { buffer: header.buffer, sh: sh };
return { buffer: header.buffer, sh: sh, shDegree: header.shDegree };
}

/**
Expand Down Expand Up @@ -1909,7 +1993,8 @@ export class GaussianSplattingMeshBase extends Mesh {
isAsync: boolean,
sh?: Uint8Array[],
partIndices?: Uint8Array,
{ flipY = false, previousVertexCount = 0 }: IUpdateOptions = {}
{ flipY = false, previousVertexCount = 0 }: IUpdateOptions = {},
shDegree?: number
): Coroutine<void> {
if (!this._covariancesATexture) {
this._readyToDisplay = false;
Expand All @@ -1935,8 +2020,8 @@ export class GaussianSplattingMeshBase extends Mesh {
this._updateSplatIndexBuffer(vertexCount);
}
this._vertexCount = vertexCount;
// degree == 1 for 1 texture (3 terms), 2 for 2 textures (8 terms) and 3 for 3 textures (15 terms)
this._shDegree = sh ? sh.length : 0;
this._maxShDegree = sh ? (shDegree ?? 0) : 0;
this._shDegree = this._maxShDegree;

const textureSize = this._getTextureSize(vertexCount);
const textureLength = textureSize.x * textureSize.y;
Expand Down Expand Up @@ -2058,10 +2143,11 @@ export class GaussianSplattingMeshBase extends Mesh {
* @param data array buffer containing center, color, orientation and scale of splats
* @param sh optional array of uint8 array for SH data
* @param partIndices optional array of uint8 for rig node indices
* @param shDegree optional SH degree of the data
* @returns a promise
*/
public async updateDataAsync(data: ArrayBuffer, sh?: Uint8Array[], partIndices?: Uint8Array): Promise<void> {
return await runCoroutineAsync(this._updateData(data, true, sh, partIndices), createYieldingScheduler());
public async updateDataAsync(data: ArrayBuffer, sh?: Uint8Array[], partIndices?: Uint8Array, shDegree?: number): Promise<void> {
return await runCoroutineAsync(this._updateData(data, true, sh, partIndices, undefined, shDegree), createYieldingScheduler());
}

/**
Expand All @@ -2071,9 +2157,10 @@ export class GaussianSplattingMeshBase extends Mesh {
* @param sh optional array of uint8 array for SH data
* @param options optional informations on how to treat data (needs to be 3rd for backward compatibility)
* @param partIndices optional array of uint8 for rig node indices
* @param shDegree optional SH degree of the data
*/
public updateData(data: ArrayBuffer, sh?: Uint8Array[], options: IUpdateOptions = { flipY: true }, partIndices?: Uint8Array): void {
runCoroutineSync(this._updateData(data, false, sh, partIndices, options));
public updateData(data: ArrayBuffer, sh?: Uint8Array[], options: IUpdateOptions = { flipY: true }, partIndices?: Uint8Array, shDegree?: number): void {
runCoroutineSync(this._updateData(data, false, sh, partIndices, options, shDegree));
}

/**
Expand Down
57 changes: 54 additions & 3 deletions packages/dev/core/src/Shaders/ShadersInclude/gaussianSplatting.fx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ struct Splat {
#if SH_DEGREE > 2
uvec4 sh2;
#endif
#if SH_DEGREE > 3
uvec4 sh3;
uvec4 sh4;
#endif
#if IS_COMPOUND
uint partIndex;
#endif
Expand Down Expand Up @@ -87,6 +91,10 @@ Splat readSplat(float splatIndex)
#if SH_DEGREE > 2
splat.sh2 = texelFetch(shTexture2, splatUVint, 0);
#endif
#if SH_DEGREE > 3
splat.sh3 = texelFetch(shTexture3, splatUVint, 0);
splat.sh4 = texelFetch(shTexture4, splatUVint, 0);
#endif
#if IS_COMPOUND
splat.partIndex = uint(texture2D(partIndicesTexture, splatUV).r * 255.0 + 0.5);
#endif
Expand All @@ -96,7 +104,7 @@ Splat readSplat(float splatIndex)
#if defined(WEBGL2) || defined(WEBGPU) || defined(NATIVE)
// no SH for GS and WebGL1
// dir = normalized(splat pos - cam pos)
vec3 computeColorFromSHDegree(vec3 dir, const vec3 sh[16])
vec3 computeColorFromSHDegree(vec3 dir, const vec3 sh[25])
{
const float SH_C0 = 0.28209479;
const float SH_C1 = 0.48860251;
Expand All @@ -116,6 +124,17 @@ vec3 computeColorFromSHDegree(vec3 dir, const vec3 sh[16])
SH_C3[5] = 1.445305721;
SH_C3[6] = -0.59004358;

float SH_C4[9];
SH_C4[0] = 2.5033429418;
SH_C4[1] = -1.7701307698;
SH_C4[2] = 0.9461746958;
SH_C4[3] = -0.6690465436;
SH_C4[4] = 0.1057855469;
SH_C4[5] = -0.6690465436;
SH_C4[6] = 0.4730873479;
SH_C4[7] = -1.7701307698;
SH_C4[8] = 0.6258357354;

vec3 result = /*SH_C0 * */sh[0];

#if SH_DEGREE > 0
Expand Down Expand Up @@ -143,6 +162,19 @@ vec3 computeColorFromSHDegree(vec3 dir, const vec3 sh[16])
SH_C3[4] * x * (4.0 * zz - xx - yy) * sh[13] +
SH_C3[5] * z * (xx - yy) * sh[14] +
SH_C3[6] * x * (xx - 3.0 * yy) * sh[15];

#if SH_DEGREE > 3
result +=
SH_C4[0] * x * y * (xx - yy) * sh[16] +
SH_C4[1] * y * z * (3.0 * xx - yy) * sh[17] +
SH_C4[2] * x * y * (7.0 * zz - 1.0) * sh[18] +
SH_C4[3] * y * z * (7.0 * zz - 3.0) * sh[19] +
SH_C4[4] * (zz * (35.0 * zz - 30.0) + 3.0) * sh[20] +
SH_C4[5] * x * z * (7.0 * zz - 3.0) * sh[21] +
SH_C4[6] * (xx - yy) * (7.0 * zz - 1.0) * sh[22] +
SH_C4[7] * x * z * (xx - 3.0 * yy) * sh[23] +
SH_C4[8] * (xx * (xx - 3.0 * yy) - yy * (3.0 * xx - yy)) * sh[24];
#endif
#endif
#endif
#endif
Expand All @@ -163,7 +195,7 @@ vec4 decompose(uint value)

vec3 computeSH(Splat splat, vec3 dir)
{
vec3 sh[16];
vec3 sh[25];

sh[0] = vec3(0.,0.,0.);
#if SH_DEGREE > 0
Expand Down Expand Up @@ -200,7 +232,26 @@ vec3 computeSH(Splat splat, vec3 dir)
sh[12] = vec3(sh08.y, sh08.z, sh08.w);
sh[13] = vec3(sh09.x, sh09.y, sh09.z);
sh[14] = vec3(sh09.w, sh10.x, sh10.y);
sh[15] = vec3(sh10.z, sh10.w, sh11.x);
sh[15] = vec3(sh10.z, sh10.w, sh11.x);
#endif
#if SH_DEGREE > 3
// sh[16] R/G/B are in sh11.y/z/w (j=45,46,47 — last 3 bytes of texture2)
vec4 sh12 = decompose(splat.sh3.x);
vec4 sh13 = decompose(splat.sh3.y);
vec4 sh14 = decompose(splat.sh3.z);
vec4 sh15 = decompose(splat.sh3.w);
vec4 sh16 = decompose(splat.sh4.x);
vec4 sh17 = decompose(splat.sh4.y);

sh[16] = vec3(sh11.y, sh11.z, sh11.w);
sh[17] = vec3(sh12.x, sh12.y, sh12.z);
sh[18] = vec3(sh12.w, sh13.x, sh13.y);
sh[19] = vec3(sh13.z, sh13.w, sh14.x);
sh[20] = vec3(sh14.y, sh14.z, sh14.w);
sh[21] = vec3(sh15.x, sh15.y, sh15.z);
sh[22] = vec3(sh15.w, sh16.x, sh16.y);
sh[23] = vec3(sh16.z, sh16.w, sh17.x);
sh[24] = vec3(sh17.y, sh17.z, sh17.w);
#endif

return computeColorFromSHDegree(dir, sh);
Expand Down
4 changes: 4 additions & 0 deletions packages/dev/core/src/Shaders/gaussianSplatting.vertex.fx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ uniform highp usampler2D shTexture1;
#if SH_DEGREE > 2
uniform highp usampler2D shTexture2;
#endif
#if SH_DEGREE > 3
uniform highp usampler2D shTexture3;
uniform highp usampler2D shTexture4;
#endif

#if IS_COMPOUND
uniform sampler2D partIndicesTexture;
Expand Down
Loading
Loading