SGD batching improvements, LBFGS and a Gaussian Process

Classification/SGD/src/main/java/org/tribuo/classification/sgd/LabelObjective.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,7 @@
 
 package org.tribuo.classification.sgd;
 
-import com.oracle.labs.mlrg.olcut.util.Pair;
 import org.tribuo.common.sgd.SGDObjective;
-import org.tribuo.math.la.SGDVector;
 import org.tribuo.math.util.VectorNormalizer;
 
 /**
@@ -27,23 +25,7 @@
  * An objective knows if it generates a probabilistic model or not,
  * and what kind of normalization needs to be applied to produce probability values.
  */
-public interface LabelObjective extends SGDObjective<Integer> {
-
-    /**
-     * Scores a prediction, returning the loss and a vector of per label gradients.
-     *
-     * @deprecated In 4.1, to migrate to the new name {@link #lossAndGradient}.
-     * @param truth      The true label id.
-     * @param prediction The prediction for each label id.
-     * @return The score and per label gradient.
-     */
-    @Deprecated
-    Pair<Double, SGDVector> valueAndGradient(int truth, SGDVector prediction);
-
-    @Override
-    default Pair<Double, SGDVector> lossAndGradient(Integer truth, SGDVector prediction) {
-        return valueAndGradient(truth, prediction);
-    }
+public interface LabelObjective extends SGDObjective<Integer, int[]> {
 
     /**
      * Generates a new {@link VectorNormalizer} which normalizes the predictions into [0,1].

Classification/SGD/src/main/java/org/tribuo/classification/sgd/crf/CRFTrainer.java

@@ -243,6 +243,19 @@ public int getInvocationCount() {
         return trainInvocationCounter;
     }
 
+    public synchronized void setInvocationCount(int invocationCount){
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        rng = new SplittableRandom(seed);
+
+        for (trainInvocationCounter = 0; trainInvocationCounter < invocationCount; trainInvocationCounter++){
+            SplittableRandom localRNG = rng.split();
+        }
+
+    }
+
     @Override
     public String toString() {
         return "CRFTrainer(optimiser="+optimiser.toString()+",epochs="+epochs+",minibatchSize="+minibatchSize+",seed="+seed+")";

Classification/SGD/src/main/java/org/tribuo/classification/sgd/fm/FMClassificationTrainer.java

@@ -40,7 +40,7 @@
  * 2010 IEEE International Conference on Data Mining
  * </pre>
  */
-public class FMClassificationTrainer extends AbstractFMTrainer<Label, Integer, FMClassificationModel> {
+public class FMClassificationTrainer extends AbstractFMTrainer<Label, Integer, FMClassificationModel, int[]> {
     private static final Logger logger = Logger.getLogger(FMClassificationTrainer.class.getName());
 
     @Config(description = "The classification objective function to use.")
@@ -108,16 +108,30 @@ private FMClassificationTrainer() {
         super();
     }
 
+    @Override
+    protected Integer[] createTargetArray(int size) {
+        return new Integer[size];
+    }
+
     @Override
     protected Integer getTarget(ImmutableOutputInfo<Label> outputInfo, Label output) {
         return outputInfo.getID(output);
     }
 
     @Override
-    protected SGDObjective<Integer> getObjective() {
+    protected LabelObjective getObjective() {
         return objective;
     }
 
+    @Override
+    protected int[] getTargetBatch(Integer[] outputs, int start, int size) {
+        int[] output = new int[size];
+        for (int i = start; i < start+size; i++) {
+            output[i - start] = outputs[i];
+        }
+        return output;
+    }
+
     @Override
     protected FMClassificationModel createModel(String name, ModelProvenance provenance, ImmutableFeatureMap featureMap, ImmutableOutputInfo<Label> outputInfo, FMParameters parameters) {
         return new FMClassificationModel(name, provenance, featureMap, outputInfo, parameters, objective.getNormalizer(), objective.isProbabilistic());

Classification/SGD/src/main/java/org/tribuo/classification/sgd/linear/LinearSGDTrainer.java

@@ -40,7 +40,7 @@
  * Proceedings of COMPSTAT, 2010.
  * </pre>
  */
-public class LinearSGDTrainer extends AbstractLinearSGDTrainer<Label,Integer,LinearSGDModel> {
+public class LinearSGDTrainer extends AbstractLinearSGDTrainer<Label,Integer,LinearSGDModel,int[]> {
     private static final Logger logger = Logger.getLogger(LinearSGDTrainer.class.getName());
 
     @Config(description = "The classification objective function to use.")
@@ -97,16 +97,30 @@ private LinearSGDTrainer() {
         super();
     }
 
+    @Override
+    protected Integer[] createTargetArray(int size) {
+        return new Integer[size];
+    }
+
     @Override
     protected Integer getTarget(ImmutableOutputInfo<Label> outputInfo, Label output) {
         return outputInfo.getID(output);
     }
 
     @Override
-    protected SGDObjective<Integer> getObjective() {
+    protected LabelObjective getObjective() {
         return objective;
     }
 
+    @Override
+    protected int[] getTargetBatch(Integer[] outputs, int start, int size) {
+        int[] output = new int[size];
+        for (int i = start; i < start+size; i++) {
+            output[i - start] = outputs[i];
+        }
+        return output;
+    }
+
     @Override
     protected LinearSGDModel createModel(String name, ModelProvenance provenance, ImmutableFeatureMap featureMap, ImmutableOutputInfo<Label> outputInfo, LinearParameters parameters) {
         return new LinearSGDModel(name, provenance, featureMap, outputInfo, parameters, objective.getNormalizer(), objective.isProbabilistic());

Classification/SGD/src/main/java/org/tribuo/classification/sgd/linear/LinearTrainer.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tribuo.classification.sgd.linear;
+
+import com.oracle.labs.mlrg.olcut.config.Config;
+import org.tribuo.Dataset;
+import org.tribuo.Example;
+import org.tribuo.ImmutableFeatureMap;
+import org.tribuo.ImmutableOutputInfo;
+import org.tribuo.classification.Label;
+import org.tribuo.classification.sgd.LabelObjective;
+import org.tribuo.classification.sgd.objectives.LogMulticlass;
+import org.tribuo.common.sgd.AbstractLinearTrainer;
+import org.tribuo.common.sgd.SGDObjective;
+import org.tribuo.math.LinearParameters;
+import org.tribuo.provenance.ModelProvenance;
+
+import java.util.logging.Logger;
+
+/**
+ * A trainer for a linear classifier using L-BFGS.
+ * <p>
+ * See:
+ * <pre>
+ * Nocedal, J. and Wright, S.
+ * "Numerical Optimization (2nd Edition)"
+ * Springer, 2006.
+ * </pre>
+ */
+public final class LinearTrainer extends AbstractLinearTrainer<Label, int[], LinearSGDModel> {
+    private static final Logger logger = Logger.getLogger(LinearTrainer.class.getName());
+
+    @Config(description = "The classification objective function to use.")
+    private LabelObjective objective = new LogMulticlass();
+
+    /**
+     * Constructs a trainer for a linear model using L-BFGS.
+     *
+     * @param objective       The objective function to optimise.
+     * @param maxIterations The maximum number of L-BFGS iterations.
+     * @param l2Penalty Should it use L2 regularisation to fit the model.
+     * @param tolerance Convergence tolerance on the loss.
+     * @param gradientTolerance Convergence tolerance on the gradient.
+     * @param regularisationStrength Strength of the L2 regularisation penalty term.
+     */
+    public LinearTrainer(LabelObjective objective, int maxIterations, boolean l2Penalty, double tolerance, double gradientTolerance, double regularisationStrength) {
+        super(maxIterations, l2Penalty, tolerance, gradientTolerance, regularisationStrength);
+        this.objective = objective;
+        postConfig();
+    }
+
+    /**
+     * For OLCUT.
+     */
+    private LinearTrainer() {
+        super();
+    }
+
+    @Override
+    public void postConfig() { }
+
+    @Override
+    public String toString() {
+        return "LinearTrainer(" +
+                "objective=" + objective +
+                ", maxIterations=" + maxIterations +
+                ", l2Penalty=" + l2Penalty +
+                ", tolerance=" + tolerance +
+                ", gradientTolerance=" + gradientTolerance +
+                ", regularisationStrength=" + regularisationStrength +
+                ", memorySize=" + memorySize +
+                ')';
+    }
+
+    @Override
+    protected int[] createTargets(Dataset<Label> dataset, ImmutableOutputInfo<Label> outputInfo) {
+        int[] outputs = new int[dataset.size()];
+
+        int i = 0;
+        for (Example<Label> e : dataset) {
+            outputs[i] = outputInfo.getID(e.getOutput());
+            i++;
+        }
+        return outputs;
+    }
+
+    @Override
+    protected SGDObjective<Integer, int[]> getObjective() {
+        return objective;
+    }
+
+    @Override
+    protected LinearSGDModel createModel(ModelProvenance provenance, ImmutableFeatureMap featureMap, ImmutableOutputInfo<Label> outputInfo, LinearParameters parameters) {
+        return new LinearSGDModel("linear-lbfgs-model", provenance, featureMap, outputInfo, parameters, objective.getNormalizer(), objective.isProbabilistic());
+    }
+
+    @Override
+    protected String getModelClassName() {
+        return LinearSGDModel.class.getName();
+    }
+}

Classification/SGD/src/main/java/org/tribuo/classification/sgd/linear/LogisticRegressionTrainer.java

@@ -20,8 +20,6 @@
 import org.tribuo.classification.sgd.objectives.LogMulticlass;
 import org.tribuo.math.optimisers.AdaGrad;
 
-import java.util.logging.Logger;
-
 /**
  * A logistic regression trainer that uses a reasonable objective, optimiser,
  * number of epochs and minibatch size. If you wish to modify any of these 
@@ -30,13 +28,12 @@
  * This is strictly a convenience class for folks who are looking for
  * a simple logistic regression.
  */
-public class LogisticRegressionTrainer extends LinearSGDTrainer {
-    private static final Logger logger = Logger.getLogger(LogisticRegressionTrainer.class.getName());
+public final class LogisticRegressionTrainer extends LinearSGDTrainer {
 
     /**
      * Constructs a simple logistic regression, using {@link AdaGrad} with a learning rate of 1.0 as
      * the gradient optimizer, training for 5 epochs.
-     *
+     * <p>
      * It's equivalent to this:
      * {@code new LinearSGDTrainer(new LogMulticlass(), new AdaGrad(1.0, 0.1), 5, Trainer.DEFAULT_SEED); }
      */