vertexfilter: Clamp exponential filter to finite range

demo/tests.cpp

@@ -1098,6 +1098,34 @@ static void encodeFilterExpOverflow()
 	assert(decoded[1] == -1.f);
 	assert(decoded[2] == 255.99997f);
 	assert(decoded[3] == 1.f);
+
+	// values near the finite float limit require exponent and mantissa saturation
+	const float large[4] = {
+	    3.402823466e+38f,
+	    1.f,
+	    -3.402823466e+38f,
+	    -1.f,
+	};
+	const unsigned int expected_large[4] = {
+	    0x7f000001,
+	    0x7f000000,
+	    0x7fffffff,
+	    0x7f000000,
+	};
+
+	unsigned int encoded_large[4];
+	meshopt_encodeFilterExp(encoded_large, 1, 16, 1, large, meshopt_EncodeExpSharedVector);
+
+	assert(memcmp(encoded_large, expected_large, sizeof(expected_large)) == 0);
+
+	float decoded_large[4];
+	memcpy(decoded_large, encoded_large, sizeof(decoded_large));
+	meshopt_decodeFilterExp(decoded_large, 1, 16);
+
+	assert(decoded_large[0] == ldexpf(1.f, 127));
+	assert(decoded_large[1] == 0.f);
+	assert(decoded_large[2] == -ldexpf(1.f, 127));
+	assert(decoded_large[3] == 0.f);
 }
 
 static void encodeFilterExpZeroShared()

src/vertexfilter.cpp

@@ -1437,16 +1437,19 @@ void meshopt_encodeFilterExp(void* destination_, size_t count, size_t stride, in
 
 			// note that we additionally scale the mantissa to make it a K-bit signed integer (K-1 bits for magnitude)
 			exp -= (bits - 1);
+			exp = exp > 127 ? 127 : exp;
 
 			// compute renormalized rounded mantissa for each component
 			const int mmask = (1 << 24) - 1;
 			const int mmax = mmask >> 1;
-
-			int m = int(v[j] * optexp2(-exp) + (v[j] >= 0 ? 0.5f : -0.5f));
-
-			// clamp mantissa so that rounding away from zero doesn't overflow
-			m = m > mmax ? mmax : m;
-
+			const int mmaxf = exp > 105 ? mmax >> (exp - 105) : mmax;
+			const int mminf = exp > 104 ? -mmaxf : -mmax - 1;
+
+			// split 2^-127 into normal factors to avoid dependence on denormal handling mode
+			float mf = exp == 127 ? v[j] * optexp2(-126) * 0.5f : v[j] * optexp2(-exp);
+			mf += (v[j] >= 0 ? 0.5f : -0.5f);
+			// clamp mantissa so that rounding away from zero doesn't overflow the storage or decoded float
+			int m = mf > mmaxf ? mmaxf : (mf < mminf ? mminf : int(mf));
 			d[j] = (m & mmask) | (unsigned(exp) << 24);
 		}
 	}