Search in sources :

Example 1 with QuadraticSearch

use of com.simiacryptus.mindseye.opt.line.QuadraticSearch in project MindsEye by SimiaCryptus.

the class LBFGSTest method train.

@Override
public void train(@Nonnull final NotebookOutput log, @Nonnull final Layer network, @Nonnull final Tensor[][] trainingData, final TrainingMonitor monitor) {
    log.code(() -> {
        @Nonnull final SimpleLossNetwork supervisedNetwork = new SimpleLossNetwork(network, new EntropyLossLayer());
        @Nonnull ValidatingTrainer trainer = new ValidatingTrainer(new SampledArrayTrainable(trainingData, supervisedNetwork, 1000, 10000), new ArrayTrainable(trainingData, supervisedNetwork).cached()).setMonitor(monitor);
        trainer.getRegimen().get(0).setOrientation(new LBFGS()).setLineSearchFactory(name -> name.toString().contains("LBFGS") ? new QuadraticSearch().setCurrentRate(1.0) : new QuadraticSearch());
        return trainer.setTimeout(5, TimeUnit.MINUTES).setMaxIterations(500).run();
    });
}
Also used : Nonnull(javax.annotation.Nonnull) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) EntropyLossLayer(com.simiacryptus.mindseye.layers.java.EntropyLossLayer) ValidatingTrainer(com.simiacryptus.mindseye.opt.ValidatingTrainer) ArrayTrainable(com.simiacryptus.mindseye.eval.ArrayTrainable) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) SimpleLossNetwork(com.simiacryptus.mindseye.network.SimpleLossNetwork)

Example 2 with QuadraticSearch

use of com.simiacryptus.mindseye.opt.line.QuadraticSearch in project MindsEye by SimiaCryptus.

the class RecursiveSubspaceTest method train.

@Override
public void train(@Nonnull final NotebookOutput log, @Nonnull final Layer network, @Nonnull final Tensor[][] trainingData, final TrainingMonitor monitor) {
    log.code(() -> {
        @Nonnull final SimpleLossNetwork supervisedNetwork = new SimpleLossNetwork(network, new EntropyLossLayer());
        @Nonnull ValidatingTrainer trainer = new ValidatingTrainer(new SampledArrayTrainable(trainingData, supervisedNetwork, 1000, 1000), new ArrayTrainable(trainingData, supervisedNetwork, 1000).cached()).setMonitor(monitor);
        trainer.getRegimen().get(0).setOrientation(getOrientation()).setLineSearchFactory(name -> name.toString().contains("LBFGS") ? new StaticLearningRate(1.0) : new QuadraticSearch());
        return trainer.setTimeout(15, TimeUnit.MINUTES).setMaxIterations(500).run();
    });
}
Also used : Nonnull(javax.annotation.Nonnull) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) StaticLearningRate(com.simiacryptus.mindseye.opt.line.StaticLearningRate) EntropyLossLayer(com.simiacryptus.mindseye.layers.java.EntropyLossLayer) ValidatingTrainer(com.simiacryptus.mindseye.opt.ValidatingTrainer) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) ArrayTrainable(com.simiacryptus.mindseye.eval.ArrayTrainable) SimpleLossNetwork(com.simiacryptus.mindseye.network.SimpleLossNetwork)

Example 3 with QuadraticSearch

use of com.simiacryptus.mindseye.opt.line.QuadraticSearch in project MindsEye by SimiaCryptus.

the class TrainingTester method trainMagic.

/**
 * Train lbfgs list.
 *
 * @param log       the log
 * @param trainable the trainable
 * @return the list
 */
@Nonnull
public List<StepRecord> trainMagic(@Nonnull final NotebookOutput log, final Trainable trainable) {
    log.p("Now we train using an experimental optimizer:");
    @Nonnull final List<StepRecord> history = new ArrayList<>();
    @Nonnull final TrainingMonitor monitor = TrainingTester.getMonitor(history);
    try {
        log.code(() -> {
            return new IterativeTrainer(trainable).setLineSearchFactory(label -> new StaticLearningRate(1.0)).setOrientation(new RecursiveSubspace() {

                @Override
                public void train(@Nonnull TrainingMonitor monitor, Layer macroLayer) {
                    @Nonnull Tensor[][] nullData = { { new Tensor() } };
                    @Nonnull BasicTrainable inner = new BasicTrainable(macroLayer);
                    @Nonnull ArrayTrainable trainable1 = new ArrayTrainable(inner, nullData);
                    inner.freeRef();
                    new IterativeTrainer(trainable1).setOrientation(new QQN()).setLineSearchFactory(n -> new QuadraticSearch().setCurrentRate(n.equals(QQN.CURSOR_NAME) ? 1.0 : 1e-4)).setMonitor(new TrainingMonitor() {

                        @Override
                        public void log(String msg) {
                            monitor.log("\t" + msg);
                        }
                    }).setMaxIterations(getIterations()).setIterationsPerSample(getIterations()).runAndFree();
                    trainable1.freeRef();
                    for (@Nonnull Tensor[] tensors : nullData) {
                        for (@Nonnull Tensor tensor : tensors) {
                            tensor.freeRef();
                        }
                    }
                }
            }).setMonitor(monitor).setTimeout(30, TimeUnit.SECONDS).setIterationsPerSample(100).setMaxIterations(250).setTerminateThreshold(0).runAndFree();
        });
    } catch (Throwable e) {
        if (isThrowExceptions())
            throw new RuntimeException(e);
    }
    return history;
}
Also used : PipelineNetwork(com.simiacryptus.mindseye.network.PipelineNetwork) IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) BiFunction(java.util.function.BiFunction) LoggerFactory(org.slf4j.LoggerFactory) Tensor(com.simiacryptus.mindseye.lang.Tensor) HashMap(java.util.HashMap) Random(java.util.Random) Result(com.simiacryptus.mindseye.lang.Result) ArmijoWolfeSearch(com.simiacryptus.mindseye.opt.line.ArmijoWolfeSearch) ArrayList(java.util.ArrayList) Trainable(com.simiacryptus.mindseye.eval.Trainable) ConstantResult(com.simiacryptus.mindseye.lang.ConstantResult) TrainingMonitor(com.simiacryptus.mindseye.opt.TrainingMonitor) Map(java.util.Map) Layer(com.simiacryptus.mindseye.lang.Layer) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) LBFGS(com.simiacryptus.mindseye.opt.orient.LBFGS) RecursiveSubspace(com.simiacryptus.mindseye.opt.orient.RecursiveSubspace) StepRecord(com.simiacryptus.mindseye.test.StepRecord) NotebookOutput(com.simiacryptus.util.io.NotebookOutput) ReferenceCounting(com.simiacryptus.mindseye.lang.ReferenceCounting) IterativeTrainer(com.simiacryptus.mindseye.opt.IterativeTrainer) Nonnull(javax.annotation.Nonnull) Nullable(javax.annotation.Nullable) MeanSqLossLayer(com.simiacryptus.mindseye.layers.java.MeanSqLossLayer) Logger(org.slf4j.Logger) PlotCanvas(smile.plot.PlotCanvas) QQN(com.simiacryptus.mindseye.opt.orient.QQN) GradientDescent(com.simiacryptus.mindseye.opt.orient.GradientDescent) BasicTrainable(com.simiacryptus.mindseye.eval.BasicTrainable) StaticLearningRate(com.simiacryptus.mindseye.opt.line.StaticLearningRate) TestUtil(com.simiacryptus.mindseye.test.TestUtil) DAGNode(com.simiacryptus.mindseye.network.DAGNode) DoubleStream(java.util.stream.DoubleStream) java.awt(java.awt) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) Stream(java.util.stream.Stream) ArrayTrainable(com.simiacryptus.mindseye.eval.ArrayTrainable) TensorList(com.simiacryptus.mindseye.lang.TensorList) Step(com.simiacryptus.mindseye.opt.Step) ProblemRun(com.simiacryptus.mindseye.test.ProblemRun) javax.swing(javax.swing) RecursiveSubspace(com.simiacryptus.mindseye.opt.orient.RecursiveSubspace) BasicTrainable(com.simiacryptus.mindseye.eval.BasicTrainable) IterativeTrainer(com.simiacryptus.mindseye.opt.IterativeTrainer) Tensor(com.simiacryptus.mindseye.lang.Tensor) Nonnull(javax.annotation.Nonnull) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) ArrayList(java.util.ArrayList) ArrayTrainable(com.simiacryptus.mindseye.eval.ArrayTrainable) Layer(com.simiacryptus.mindseye.lang.Layer) MeanSqLossLayer(com.simiacryptus.mindseye.layers.java.MeanSqLossLayer) QQN(com.simiacryptus.mindseye.opt.orient.QQN) StepRecord(com.simiacryptus.mindseye.test.StepRecord) TrainingMonitor(com.simiacryptus.mindseye.opt.TrainingMonitor) StaticLearningRate(com.simiacryptus.mindseye.opt.line.StaticLearningRate) Nonnull(javax.annotation.Nonnull)

Example 4 with QuadraticSearch

use of com.simiacryptus.mindseye.opt.line.QuadraticSearch in project MindsEye by SimiaCryptus.

the class ImageDecompositionLab method train.

/**
 * Train.
 *
 * @param log            the log
 * @param monitor        the monitor
 * @param network        the network
 * @param data           the data
 * @param timeoutMinutes the timeout minutes
 * @param mask           the mask
 */
protected void train(@Nonnull final NotebookOutput log, final TrainingMonitor monitor, final Layer network, @Nonnull final Tensor[][] data, final int timeoutMinutes, final boolean... mask) {
    log.out("Training for %s minutes, mask=%s", timeoutMinutes, Arrays.toString(mask));
    log.code(() -> {
        @Nonnull SampledTrainable trainingSubject = new SampledArrayTrainable(data, network, data.length);
        trainingSubject = (SampledTrainable) ((TrainableDataMask) trainingSubject).setMask(mask);
        @Nonnull final ValidatingTrainer validatingTrainer = new ValidatingTrainer(trainingSubject, new ArrayTrainable(data, network)).setMaxTrainingSize(data.length).setMinTrainingSize(5).setMonitor(monitor).setTimeout(timeoutMinutes, TimeUnit.MINUTES).setMaxIterations(1000);
        validatingTrainer.getRegimen().get(0).setOrientation(new GradientDescent()).setLineSearchFactory(name -> name.equals(QQN.CURSOR_NAME) ? new QuadraticSearch().setCurrentRate(1.0) : new QuadraticSearch().setCurrentRate(1.0));
        validatingTrainer.run();
    });
}
Also used : TrainableDataMask(com.simiacryptus.mindseye.eval.TrainableDataMask) SampledTrainable(com.simiacryptus.mindseye.eval.SampledTrainable) Nonnull(javax.annotation.Nonnull) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) GradientDescent(com.simiacryptus.mindseye.opt.orient.GradientDescent) ValidatingTrainer(com.simiacryptus.mindseye.opt.ValidatingTrainer) SampledArrayTrainable(com.simiacryptus.mindseye.eval.SampledArrayTrainable) ArrayTrainable(com.simiacryptus.mindseye.eval.ArrayTrainable)

Example 5 with QuadraticSearch

use of com.simiacryptus.mindseye.opt.line.QuadraticSearch in project MindsEye by SimiaCryptus.

the class TrainingTester method trainCjGD.

/**
 * Train cj gd list.
 *
 * @param log       the log
 * @param trainable the trainable
 * @return the list
 */
@Nonnull
public List<StepRecord> trainCjGD(@Nonnull final NotebookOutput log, final Trainable trainable) {
    log.p("First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.");
    @Nonnull final List<StepRecord> history = new ArrayList<>();
    @Nonnull final TrainingMonitor monitor = TrainingTester.getMonitor(history);
    try {
        log.code(() -> {
            return new IterativeTrainer(trainable).setLineSearchFactory(label -> new QuadraticSearch()).setOrientation(new GradientDescent()).setMonitor(monitor).setTimeout(30, TimeUnit.SECONDS).setMaxIterations(250).setTerminateThreshold(0).runAndFree();
        });
    } catch (Throwable e) {
        if (isThrowExceptions())
            throw new RuntimeException(e);
    }
    return history;
}
Also used : StepRecord(com.simiacryptus.mindseye.test.StepRecord) TrainingMonitor(com.simiacryptus.mindseye.opt.TrainingMonitor) IterativeTrainer(com.simiacryptus.mindseye.opt.IterativeTrainer) Nonnull(javax.annotation.Nonnull) QuadraticSearch(com.simiacryptus.mindseye.opt.line.QuadraticSearch) ArrayList(java.util.ArrayList) GradientDescent(com.simiacryptus.mindseye.opt.orient.GradientDescent) Nonnull(javax.annotation.Nonnull)

Aggregations

QuadraticSearch (com.simiacryptus.mindseye.opt.line.QuadraticSearch)5 Nonnull (javax.annotation.Nonnull)5 ArrayTrainable (com.simiacryptus.mindseye.eval.ArrayTrainable)4 SampledArrayTrainable (com.simiacryptus.mindseye.eval.SampledArrayTrainable)3 ValidatingTrainer (com.simiacryptus.mindseye.opt.ValidatingTrainer)3 GradientDescent (com.simiacryptus.mindseye.opt.orient.GradientDescent)3 EntropyLossLayer (com.simiacryptus.mindseye.layers.java.EntropyLossLayer)2 SimpleLossNetwork (com.simiacryptus.mindseye.network.SimpleLossNetwork)2 IterativeTrainer (com.simiacryptus.mindseye.opt.IterativeTrainer)2 TrainingMonitor (com.simiacryptus.mindseye.opt.TrainingMonitor)2 StaticLearningRate (com.simiacryptus.mindseye.opt.line.StaticLearningRate)2 StepRecord (com.simiacryptus.mindseye.test.StepRecord)2 ArrayList (java.util.ArrayList)2 BasicTrainable (com.simiacryptus.mindseye.eval.BasicTrainable)1 SampledTrainable (com.simiacryptus.mindseye.eval.SampledTrainable)1 Trainable (com.simiacryptus.mindseye.eval.Trainable)1 TrainableDataMask (com.simiacryptus.mindseye.eval.TrainableDataMask)1 ConstantResult (com.simiacryptus.mindseye.lang.ConstantResult)1 Layer (com.simiacryptus.mindseye.lang.Layer)1 ReferenceCounting (com.simiacryptus.mindseye.lang.ReferenceCounting)1