Add new experimental 'window' regression function.

Author: vmiura

MotionMark/developer.html

@@ -113,6 +113,8 @@ <h3>Adjusting the test complexity:</h3>
                                         <select name="score-profile" id="score-profile">
                                             <option value="slope">Slope</option>
                                             <option value="flat">Flat</option>
+                                            <option value="window">Windowed</option>
+                                            <option value="window-strict">Windowed Strict</option>
                                         </select>
                                     </label>
                                     </li>
@@ -165,6 +167,8 @@ <h1>MotionMark score</h1>
                         <select id="results-score-profile" onchange="benchmarkController.changeScoreProfile(this.value)">
                             <option value="slope">Slope</option>
                             <option value="flat">Flat</option>
+                            <option value="window">Windowed</option>
+                            <option value="window-strict">Windowed Strict</option>
                         </select>
                     </label>
                 </div>
@@ -195,6 +199,8 @@ <h1>Graph:</h1>
                             <select id="graph-score-profile" onchange="benchmarkController.changeScoreProfile(this.value)">
                                 <option value="slope">Slope</option>
                                 <option value="flat">Flat</option>
+                                <option value="window">Windowed</option>
+                                <option value="window-strict">Windowed Strict</option>
                             </select>
                         </label>
                     </div>

MotionMark/resources/runner/results.js

@@ -187,14 +187,15 @@ class ScoreCalculator {
 
             const frameTypeIndex = series.fieldMap[Strings.json.frameType];
             const complexityIndex = series.fieldMap[complexityKey];
+            const frameTimeIndex = series.fieldMap[Strings.json.time];
             const frameLengthIndex = series.fieldMap[Strings.json.frameLength];
             const regressionOptions = { desiredFrameLength: desiredFrameLength };
             if (profile)
                 regressionOptions.preferredProfile = profile;
 
             const regressionSamples = series.slice(minIndex, maxIndex + 1);
             const animationSamples = regressionSamples.data.filter((sample) => sample[frameTypeIndex] == Strings.json.animationFrameType);
-            const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex] ]);
+            const regressionData = animationSamples.map((sample) => [ sample[complexityIndex], sample[frameLengthIndex], sample[frameTimeIndex] ]);
 
             const regression = new Regression(regressionData, minIndex, maxIndex, regressionOptions);
             return {

MotionMark/resources/statistics.js

@@ -178,8 +178,10 @@ class Regression {
     constructor(samples, startIndex, endIndex, options)
     {
         const desiredFrameLength = options.desiredFrameLength;
-        var profile;
+        const kWindowSizeMultiple = 0.1;
 
+        var profile;
+            
         if (!options.preferredProfile || options.preferredProfile == Strings.json.profiles.slope) {
             profile = this._calculateRegression(samples, {
                 shouldClip: true,
@@ -195,6 +197,11 @@ class Regression {
                 t2: 0
             });
             this.profile = Strings.json.profiles.flat;
+        } else if (options.preferredProfile == Strings.json.profiles.window || options.preferredProfile == Strings.json.profiles.windowStrict) {
+            const window_size = Math.max(1, Math.floor(samples.length * kWindowSizeMultiple));
+            const strict = options.preferredProfile == Strings.json.profiles.windowStrict;
+            profile = this._windowedFit(samples, desiredFrameLength, window_size, strict);
+            this.profile = options.preferredProfile;
         }
 
         this.startIndex = Math.min(startIndex, endIndex);
@@ -219,6 +226,111 @@ class Regression {
         return this.s1 + this.t1 * complexity;
     }
 
+    _windowedFit(samples, desiredFrameLength, windowSize, strict)
+    {
+        const kAllowedErrorFactor = 0.9;
+
+        const complexityIndex = 0;
+        const frameLengthIndex = 1;
+        const frameTimeIndex = 2;
+
+        const average = array => array.reduce((a, b) => a + b) / array.length;
+
+        var sortedSamples = samples.slice().sort((a, b) => {
+                if (a[complexityIndex] == b[complexityIndex])
+                    return b[frameTimeIndex] - a[frameTimeIndex];
+                return a[complexityIndex] - b[complexityIndex];
+            });
+
+        var cumFrameLength = 0.0;
+        var bestIndex = 0;
+        var bestComplexity = 0;
+        var runningFrameLengths = [];
+        var runningComplexities = [];
+
+        for (var i = 0; i < sortedSamples.length; ++i) {
+            runningFrameLengths.push(sortedSamples[i][frameLengthIndex]);
+            runningComplexities.push(sortedSamples[i][complexityIndex]);
+
+            if (runningFrameLengths.length < windowSize) {
+                continue
+            } else if (runningFrameLengths.length > windowSize) {
+                runningFrameLengths.shift();
+                runningComplexities.shift();
+            }
+
+            let averageFrameLength = average(runningFrameLengths);
+            let averageComplexity = average(runningComplexities);
+            let error = desiredFrameLength / averageFrameLength;
+            let adjustedComplexity = averageComplexity * Math.min(1.0, error);
+
+            if (error >= kAllowedErrorFactor) {
+                if (adjustedComplexity > bestComplexity) {
+                    bestComplexity = adjustedComplexity;
+                }
+            } else if (strict) {
+                break;
+            }
+        }
+
+        for (var i = 0; i < sortedSamples.length; ++i) {
+            if (sortedSamples[i][complexityIndex] <= bestComplexity)
+                bestIndex = i;
+        }
+
+        let complexity = bestComplexity;
+
+        // Calculate slope for remaining points
+        let t_nom = 0.0;
+        let t_denom = 0.0;
+        for (var i = bestIndex + 1; i < sortedSamples.length; i++) {
+            const tx = sortedSamples[i][complexityIndex] - complexity;
+            const ty = sortedSamples[i][frameLengthIndex] - desiredFrameLength;
+            t_nom += tx * ty;
+            t_denom += tx * tx;
+        }
+
+        var s1 = desiredFrameLength;
+        var t1 = 0;
+        var t2 = (t_nom / t_denom) || 0.0;
+        var s2 = desiredFrameLength - t2 * complexity;
+        var n1 = bestIndex + 1;
+        var n2 = sortedSamples.length - bestIndex - 1;
+
+        function getValueAt(at_complexity)
+        {
+            if (at_complexity > complexity)
+                return s2 + t2 * complexity;
+            return s1 + t1 * complexity;
+        }
+
+        let error1 = 0.0;
+        let error2 = 0.0;
+        for (var i = 0; i < n1; ++i) {
+            const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+            error1 += frameLengthErr * frameLengthErr;
+        }
+        for (var i = n1; i < sortedSamples.length; ++i) {
+            const frameLengthErr = sortedSamples[i][frameLengthIndex] - getValueAt(sortedSamples[i][complexityIndex]);
+            error2 += frameLengthErr * frameLengthErr;
+        }
+        
+        return {
+            s1: s1,
+            t1: t1,
+            s2: s2,
+            t2: t2,
+            complexity: complexity,
+            // Number of samples included in the first segment, inclusive of bestIndex
+            n1: n1,
+            // Number of samples included in the second segment
+            n2: n2,
+            stdev1: Math.sqrt(error1 / n1),
+            stdev2: Math.sqrt(error2 / n2),
+            error: error1 + error2,
+        };        
+    }
+
     // A generic two-segment piecewise regression calculator. Based on Kundu/Ubhaya
     //
     // Minimize sum of (y - y')^2

MotionMark/resources/strings.js

@@ -78,7 +78,9 @@ var Strings = {
 
         profiles: {
             slope: "slope",
-            flat: "flat"
+            flat: "flat",
+            window: "window",
+            windowStrict: "window-strict",
         },
 
         results: {

MotionMark/tests/resources/controllers.js

@@ -170,10 +170,15 @@ class Controller {
     {
         return samples[sampleTimeIndex][i] - samples[sampleTimeIndex][i - 1];
     }
+
+    _getFrameTime(samples, i)
+    {
+        return samples[sampleTimeIndex][i];
+    }
 
     _previousFrameComplexity(samples, i)
     {
-        if (i > 0)
+        if (i > 1)
             return this._getComplexity(samples, i - 1);
 
         return 0;
@@ -415,6 +420,7 @@ class RampController extends Controller {
         super(benchmark, options);
 
         this.targetFPS = targetFPS;
+        this.preferredProfile = options["score-profile"];
 
         // Initially start with a tier test to find the bounds
         // The number of objects in a tier test is 10^|_tier|
@@ -613,10 +619,15 @@ class RampController extends Controller {
         for (var i = this._rampStartIndex; i < this._sampler.sampleCount; ++i) {
             if (this._getFrameType(this._sampler.samples, i) == Strings.json.mutationFrameType)
                 continue;
-            regressionData.push([ this._getComplexity(this._sampler.samples, i), this._getFrameLength(this._sampler.samples, i) ]);
+            regressionData.push(
+                [
+                    this._getComplexity(this._sampler.samples, i), 
+                    this._getFrameLength(this._sampler.samples, i),
+                    this._getFrameTime(this._sampler.samples, i)
+                ]);
         }
 
-        var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired });
+        var regression = new Regression(regressionData, this._sampler.sampleCount - 1, this._rampStartIndex, { desiredFrameLength: this.frameLengthDesired, preferredProfile: this.preferredProfile });
         this._rampRegressions.push(regression);
 
         var frameLengthAtMaxComplexity = regression.valueAt(this._maximumComplexity);