Examples with StochasticComponent com.simiacryptus.mindseye.layers.java.StochasticComponent used on opensource projects

Example 1 with StochasticComponent

use of com.simiacryptus.mindseye.layers.java.StochasticComponent in project MindsEye by SimiaCryptus.

the class ValidatingTrainer method reset.

@Nonnull
private ValidatingTrainer reset(@Nonnull final TrainingPhase phase, final long seed) {
    if (!phase.trainingSubject.reseed(seed))
        throw new IterativeStopException();
    phase.orientation.reset();
    phase.trainingSubject.reseed(seed);
    if (phase.trainingSubject.getLayer() instanceof DAGNetwork) {
        ((DAGNetwork) phase.trainingSubject.getLayer()).visitLayers(layer -> {
            if (layer instanceof StochasticComponent)
                ((StochasticComponent) layer).shuffle(StochasticComponent.random.get().nextLong());
        });
    }
    return this;
}

Example 2 with StochasticComponent

use of com.simiacryptus.mindseye.layers.java.StochasticComponent in project MindsEye by SimiaCryptus.

the class ValidatingTrainer method run.

/**
 * Run double.
 *
 * @return the double
 */
public double run() {
    try {
        final long timeoutAt = System.currentTimeMillis() + timeout.toMillis();
        if (validationSubject.getLayer() instanceof DAGNetwork) {
            ((DAGNetwork) validationSubject.getLayer()).visitLayers(layer -> {
                if (layer instanceof StochasticComponent)
                    ((StochasticComponent) layer).clearNoise();
            });
        }
        @Nonnull final EpochParams epochParams = new EpochParams(timeoutAt, epochIterations, getTrainingSize(), validationSubject.measure(monitor));
        int epochNumber = 0;
        int iterationNumber = 0;
        int lastImprovement = 0;
        double lowestValidation = Double.POSITIVE_INFINITY;
        while (true) {
            if (shouldHalt(monitor, timeoutAt)) {
                monitor.log("Training halted");
                break;
            }
            monitor.log(String.format("Epoch parameters: %s, %s", epochParams.trainingSize, epochParams.iterations));
            @Nonnull final List<TrainingPhase> regimen = getRegimen();
            final long seed = System.nanoTime();
            final List<EpochResult> epochResults = IntStream.range(0, regimen.size()).mapToObj(i -> {
                final TrainingPhase phase = getRegimen().get(i);
                return runPhase(epochParams, phase, i, seed);
            }).collect(Collectors.toList());
            final EpochResult primaryPhase = epochResults.get(0);
            iterationNumber += primaryPhase.iterations;
            final double trainingDelta = primaryPhase.currentPoint.getMean() / primaryPhase.priorMean;
            if (validationSubject.getLayer() instanceof DAGNetwork) {
                ((DAGNetwork) validationSubject.getLayer()).visitLayers(layer -> {
                    if (layer instanceof StochasticComponent)
                        ((StochasticComponent) layer).clearNoise();
                });
            }
            final PointSample currentValidation = validationSubject.measure(monitor);
            final double overtraining = Math.log(trainingDelta) / Math.log(currentValidation.getMean() / epochParams.validation.getMean());
            final double validationDelta = currentValidation.getMean() / epochParams.validation.getMean();
            final double adj1 = Math.pow(Math.log(getTrainingTarget()) / Math.log(validationDelta), adjustmentFactor);
            final double adj2 = Math.pow(overtraining / getOvertrainingTarget(), adjustmentFactor);
            final double validationMean = currentValidation.getMean();
            if (validationMean < lowestValidation) {
                lowestValidation = validationMean;
                lastImprovement = iterationNumber;
            }
            monitor.log(String.format("Epoch %d result apply %s iterations, %s/%s samples: {validation *= 2^%.5f; training *= 2^%.3f; Overtraining = %.2f}, {itr*=%.2f, len*=%.2f} %s since improvement; %.4f validation time", ++epochNumber, primaryPhase.iterations, epochParams.trainingSize, getMaxTrainingSize(), Math.log(validationDelta) / Math.log(2), Math.log(trainingDelta) / Math.log(2), overtraining, adj1, adj2, iterationNumber - lastImprovement, validatingMeasurementTime.getAndSet(0) / 1e9));
            if (!primaryPhase.continueTraining) {
                monitor.log(String.format("Training %d runPhase halted", epochNumber));
                break;
            }
            if (epochParams.trainingSize >= getMaxTrainingSize()) {
                final double roll = FastRandom.INSTANCE.random();
                if (roll > Math.pow(2 - validationDelta, pessimism)) {
                    monitor.log(String.format("Training randomly converged: %3f", roll));
                    break;
                } else {
                    if (iterationNumber - lastImprovement > improvmentStaleThreshold) {
                        if (disappointments.incrementAndGet() > getDisappointmentThreshold()) {
                            monitor.log(String.format("Training converged after %s iterations", iterationNumber - lastImprovement));
                            break;
                        } else {
                            monitor.log(String.format("Training failed to converged on %s attempt after %s iterations", disappointments.get(), iterationNumber - lastImprovement));
                        }
                    } else {
                        disappointments.set(0);
                    }
                }
            }
            if (validationDelta < 1.0 && trainingDelta < 1.0) {
                if (adj1 < 1 - adjustmentTolerance || adj1 > 1 + adjustmentTolerance) {
                    epochParams.iterations = Math.max(getMinEpochIterations(), Math.min(getMaxEpochIterations(), (int) (primaryPhase.iterations * adj1)));
                }
                if (adj2 < 1 + adjustmentTolerance || adj2 > 1 - adjustmentTolerance) {
                    epochParams.trainingSize = Math.max(0, Math.min(Math.max(getMinTrainingSize(), Math.min(getMaxTrainingSize(), (int) (epochParams.trainingSize * adj2))), epochParams.trainingSize));
                }
            } else {
                epochParams.trainingSize = Math.max(0, Math.min(Math.max(getMinTrainingSize(), Math.min(getMaxTrainingSize(), epochParams.trainingSize * 5)), epochParams.trainingSize));
                epochParams.iterations = 1;
            }
            epochParams.validation = currentValidation;
        }
        if (validationSubject.getLayer() instanceof DAGNetwork) {
            ((DAGNetwork) validationSubject.getLayer()).visitLayers(layer -> {
                if (layer instanceof StochasticComponent)
                    ((StochasticComponent) layer).clearNoise();
            });
        }
        return epochParams.validation.getMean();
    } catch (@Nonnull final Throwable e) {
        throw new RuntimeException(e);
    }
}

Example 3 with StochasticComponent

use of com.simiacryptus.mindseye.layers.java.StochasticComponent in project MindsEye by SimiaCryptus.

the class StochasticSamplingSubnetLayer method eval.

@Nullable
@Override
public Result eval(@Nonnull final Result... inObj) {
    Result[] counting = Arrays.stream(inObj).map(r -> {
        return new CountingResult(r, samples);
    }).toArray(i -> new Result[i]);
    return average(Arrays.stream(getSeeds()).mapToObj(seed -> {
        Layer inner = getInner();
        if (inner instanceof DAGNetwork) {
            ((DAGNetwork) inner).visitNodes(node -> {
                Layer layer = node.getLayer();
                if (layer instanceof StochasticComponent) {
                    ((StochasticComponent) layer).shuffle(seed);
                }
                if (layer instanceof MultiPrecision<?>) {
                    ((MultiPrecision) layer).setPrecision(precision);
                }
            });
        }
        if (inner instanceof MultiPrecision<?>) {
            ((MultiPrecision) inner).setPrecision(precision);
        }
        if (inner instanceof StochasticComponent) {
            ((StochasticComponent) inner).shuffle(seed);
        }
        inner.setFrozen(isFrozen());
        return inner.eval(counting);
    }).toArray(i -> new Result[i]), precision);
}

Example 4 with StochasticComponent

use of com.simiacryptus.mindseye.layers.java.StochasticComponent in project MindsEye by SimiaCryptus.

the class IterativeTrainer method measure.

/**
 * Measure point sample.
 *
 * @param reset the reset
 * @return the point sample
 */
@Nullable
public PointSample measure(boolean reset) {
    @Nullable PointSample currentPoint = null;
    int retries = 0;
    do {
        if (reset) {
            orientation.reset();
            if (subject.getLayer() instanceof DAGNetwork) {
                ((DAGNetwork) subject.getLayer()).visitLayers(layer -> {
                    if (layer instanceof StochasticComponent)
                        ((StochasticComponent) layer).shuffle(StochasticComponent.random.get().nextLong());
                });
            }
            if (!subject.reseed(System.nanoTime())) {
                if (retries > 0)
                    throw new IterativeStopException("Failed to reset training subject");
            } else {
                monitor.log(String.format("Reset training subject"));
            }
        }
        if (null != currentPoint) {
            currentPoint.freeRef();
        }
        currentPoint = subject.measure(monitor);
    } while (!Double.isFinite(currentPoint.getMean()) && 10 < retries++);
    if (!Double.isFinite(currentPoint.getMean())) {
        currentPoint.freeRef();
        throw new IterativeStopException();
    }
    return currentPoint;
}

Example 5 with StochasticComponent

use of com.simiacryptus.mindseye.layers.java.StochasticComponent in project MindsEye by SimiaCryptus.

the class IterativeTrainer method run.

/**
 * Run double.
 *
 * @return the double
 */
public double run() {
    final long timeoutMs = System.currentTimeMillis() + timeout.toMillis();
    long lastIterationTime = System.nanoTime();
    @Nullable PointSample currentPoint = measure(true);
    mainLoop: while (timeoutMs > System.currentTimeMillis() && currentPoint.getMean() > terminateThreshold) {
        if (currentIteration.get() > maxIterations) {
            break;
        }
        currentPoint.freeRef();
        currentPoint = measure(true);
        assert 0 < currentPoint.delta.getMap().size() : "Nothing to optimize";
        subiterationLoop: for (int subiteration = 0; subiteration < iterationsPerSample || iterationsPerSample <= 0; subiteration++) {
            if (timeoutMs < System.currentTimeMillis()) {
                break mainLoop;
            }
            if (currentIteration.incrementAndGet() > maxIterations) {
                break mainLoop;
            }
            currentPoint.freeRef();
            currentPoint = measure(true);
            @Nullable final PointSample _currentPoint = currentPoint;
            @Nonnull final TimedResult<LineSearchCursor> timedOrientation = TimedResult.time(() -> orientation.orient(subject, _currentPoint, monitor));
            final LineSearchCursor direction = timedOrientation.result;
            final CharSequence directionType = direction.getDirectionType();
            @Nullable final PointSample previous = currentPoint;
            previous.addRef();
            try {
                @Nonnull final TimedResult<PointSample> timedLineSearch = TimedResult.time(() -> step(direction, directionType, previous));
                currentPoint.freeRef();
                currentPoint = timedLineSearch.result;
                final long now = System.nanoTime();
                final CharSequence perfString = String.format("Total: %.4f; Orientation: %.4f; Line Search: %.4f", (now - lastIterationTime) / 1e9, timedOrientation.timeNanos / 1e9, timedLineSearch.timeNanos / 1e9);
                lastIterationTime = now;
                monitor.log(String.format("Fitness changed from %s to %s", previous.getMean(), currentPoint.getMean()));
                if (previous.getMean() <= currentPoint.getMean()) {
                    if (previous.getMean() < currentPoint.getMean()) {
                        monitor.log(String.format("Resetting Iteration %s", perfString));
                        currentPoint.freeRef();
                        currentPoint = direction.step(0, monitor).point;
                    } else {
                        monitor.log(String.format("Static Iteration %s", perfString));
                    }
                    if (subject.reseed(System.nanoTime())) {
                        monitor.log(String.format("Iteration %s failed, retrying. Error: %s", currentIteration.get(), currentPoint.getMean()));
                        monitor.log(String.format("Previous Error: %s -> %s", previous.getRate(), previous.getMean()));
                        break subiterationLoop;
                    } else {
                        monitor.log(String.format("Iteration %s failed, aborting. Error: %s", currentIteration.get(), currentPoint.getMean()));
                        monitor.log(String.format("Previous Error: %s -> %s", previous.getRate(), previous.getMean()));
                        break mainLoop;
                    }
                } else {
                    monitor.log(String.format("Iteration %s complete. Error: %s " + perfString, currentIteration.get(), currentPoint.getMean()));
                }
                monitor.onStepComplete(new Step(currentPoint, currentIteration.get()));
            } finally {
                previous.freeRef();
                direction.freeRef();
            }
        }
    }
    if (subject.getLayer() instanceof DAGNetwork) {
        ((DAGNetwork) subject.getLayer()).visitLayers(layer -> {
            if (layer instanceof StochasticComponent)
                ((StochasticComponent) layer).clearNoise();
        });
    }
    double mean = null == currentPoint ? Double.NaN : currentPoint.getMean();
    currentPoint.freeRef();
    return mean;
}