use of com.simiacryptus.mindseye.lang.Tensor in project MindsEye by SimiaCryptus.
the class BatchDerivativeTester method testLearning.
/**
* Test learning tolerance statistics.
*
* @param component the component
* @param IOPair the io pair
* @param statistics the statistics
* @return the tolerance statistics
*/
public ToleranceStatistics testLearning(@Nonnull Layer component, @Nonnull IOPair IOPair, ToleranceStatistics statistics) {
final ToleranceStatistics prev = statistics;
statistics = IntStream.range(0, component.state().size()).mapToObj(i -> {
@Nullable final Tensor measuredGradient = !verify ? null : measureLearningGradient(component, i, IOPair.getOutputPrototype(), IOPair.getInputPrototype());
@Nonnull final Tensor implementedGradient = getLearningGradient(component, i, IOPair.getOutputPrototype(), IOPair.getInputPrototype());
try {
final ToleranceStatistics result = IntStream.range(0, null == measuredGradient ? 0 : measuredGradient.length()).mapToObj(i1 -> {
return new ToleranceStatistics().accumulate(measuredGradient.getData()[i1], implementedGradient.getData()[i1]);
}).reduce((a, b) -> a.combine(b)).orElse(new ToleranceStatistics());
if (!(result.absoluteTol.getMax() < tolerance)) {
throw new AssertionError(result.toString());
} else {
// log.info(String.format("Component: %s", component));
if (verbose) {
log.info(String.format("Learning Gradient for weight setByCoord %s", i));
log.info(String.format("Weights: %s", new Tensor(component.state().get(i)).prettyPrint()));
log.info(String.format("Implemented Gradient: %s", implementedGradient.prettyPrint()));
log.info(String.format("Implemented Statistics: %s", new ScalarStatistics().add(implementedGradient.getData())));
if (null != measuredGradient) {
log.info(String.format("Measured Gradient: %s", measuredGradient.prettyPrint()));
log.info(String.format("Measured Statistics: %s", new ScalarStatistics().add(measuredGradient.getData())));
log.info(String.format("Gradient Error: %s", measuredGradient.minus(implementedGradient).prettyPrint()));
log.info(String.format("Error Statistics: %s", new ScalarStatistics().add(measuredGradient.minus(implementedGradient).getData())));
}
}
return result;
}
} catch (@Nonnull final Throwable e) {
// log.info(String.format("Component: %s", component));
log.info(String.format("Learning Gradient for weight setByCoord %s", i));
log.info(String.format("Implemented Gradient: %s", implementedGradient.prettyPrint()));
log.info(String.format("Implemented Statistics: %s", new ScalarStatistics().add(implementedGradient.getData())));
if (null != measuredGradient) {
log.info(String.format("Measured Gradient: %s", measuredGradient.prettyPrint()));
log.info(String.format("Measured Statistics: %s", new ScalarStatistics().add(measuredGradient.getData())));
log.info(String.format("Gradient Error: %s", measuredGradient.minus(implementedGradient).prettyPrint()));
log.info(String.format("Error Statistics: %s", new ScalarStatistics().add(measuredGradient.minus(implementedGradient).getData())));
}
throw e;
}
}).reduce((a, b) -> a.combine(b)).map(x -> x.combine(prev)).orElseGet(() -> prev);
return statistics;
}
use of com.simiacryptus.mindseye.lang.Tensor in project MindsEye by SimiaCryptus.
the class BatchDerivativeTester method testUnFrozen.
/**
* Test un frozen.
*
* @param component the component
* @param inputPrototype the input prototype
*/
public void testUnFrozen(@Nonnull final Layer component, final Tensor[] inputPrototype) {
@Nonnull final AtomicBoolean reachedInputFeedback = new AtomicBoolean(false);
@Nonnull final Layer frozen = component.copy().setFrozen(false);
@Nullable final Result eval = frozen.eval(new Result(TensorArray.create(inputPrototype), (@Nonnull final DeltaSet<Layer> buffer, @Nonnull final TensorList data) -> {
reachedInputFeedback.set(true);
}) {
@Override
public boolean isAlive() {
return true;
}
});
@Nonnull final DeltaSet<Layer> buffer = new DeltaSet<Layer>();
TensorList data = eval.getData();
eval.accumulate(buffer, data);
@Nullable final List<double[]> stateList = frozen.state();
final List<Delta<Layer>> deltas = stateList.stream().map(doubles -> {
return buffer.stream().filter(x -> x.target == doubles).findFirst().orElse(null);
}).filter(x -> x != null).collect(Collectors.toList());
if (deltas.isEmpty() && !stateList.isEmpty()) {
throw new AssertionError("Nonfrozen component not listed in delta. Deltas: " + deltas);
}
if (!reachedInputFeedback.get()) {
throw new RuntimeException("Nonfrozen component did not pass input backwards");
}
}
use of com.simiacryptus.mindseye.lang.Tensor in project MindsEye by SimiaCryptus.
the class BatchDerivativeTester method getLearningGradient.
@Nonnull
private Tensor getLearningGradient(@Nonnull final Layer component, final int layerNum, @Nonnull final Tensor outputPrototype, final Tensor... inputPrototype) {
component.setFrozen(false);
final double[] stateArray = component.state().get(layerNum);
final int stateLen = stateArray.length;
@Nonnull final Tensor gradient = new Tensor(stateLen, outputPrototype.length());
for (int j = 0; j < outputPrototype.length(); j++) {
final int j_ = j;
@Nonnull final DeltaSet<Layer> buffer = new DeltaSet<Layer>();
@Nonnull final Tensor data = new Tensor(outputPrototype.getDimensions()).set((k) -> k == j_ ? 1 : 0);
@Nullable final Result eval = component.eval(ConstantResult.singleResultArray(new Tensor[][] { inputPrototype }));
eval.getData().get(0);
@Nonnull TensorArray tensorArray = TensorArray.wrap(data);
eval.accumulate(buffer, tensorArray);
final DoubleBuffer<Layer> deltaFlushBuffer = buffer.getMap().values().stream().filter(x -> x.target == stateArray).findFirst().orElse(null);
if (null != deltaFlushBuffer) {
for (int i = 0; i < stateLen; i++) {
gradient.set(new int[] { i, j_ }, deltaFlushBuffer.getDelta()[i]);
}
}
}
return gradient;
}
use of com.simiacryptus.mindseye.lang.Tensor in project MindsEye by SimiaCryptus.
the class BatchDerivativeTester method test.
/**
* Test tolerance statistics.
*
* @param log
* @param component the component
* @param inputPrototype the input prototype
* @return the tolerance statistics
*/
@Override
public ToleranceStatistics test(@Nonnull final NotebookOutput log, @Nonnull final Layer component, @Nonnull final Tensor... inputPrototype) {
log.h1("Differential Validation");
@Nonnull IOPair ioPair = new IOPair(component, inputPrototype[0]).invoke();
if (verbose) {
log.code(() -> {
BatchDerivativeTester.log.info(String.format("Inputs: %s", Arrays.stream(inputPrototype).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).get()));
BatchDerivativeTester.log.info(String.format("Inputs Statistics: %s", Arrays.stream(inputPrototype).map(x -> new ScalarStatistics().add(x.getData()).toString()).reduce((a, b) -> a + ",\n" + b).get()));
BatchDerivativeTester.log.info(String.format("Output: %s", ioPair.getOutputPrototype().prettyPrint()));
BatchDerivativeTester.log.info(String.format("Outputs Statistics: %s", new ScalarStatistics().add(ioPair.getOutputPrototype().getData())));
});
}
ToleranceStatistics _statistics = new ToleranceStatistics();
if (isTestFeedback()) {
log.h2("Feedback Validation");
log.p("We validate the agreement between the implemented derivative _of the inputs_ apply finite difference estimations:");
ToleranceStatistics statistics = _statistics;
_statistics = log.code(() -> {
return testFeedback(component, ioPair, statistics);
});
}
if (isTestLearning()) {
log.h2("Learning Validation");
log.p("We validate the agreement between the implemented derivative _of the internal weights_ apply finite difference estimations:");
ToleranceStatistics statistics = _statistics;
_statistics = log.code(() -> {
return testLearning(component, ioPair, statistics);
});
}
log.h2("Total Accuracy");
log.p("The overall agreement accuracy between the implemented derivative and the finite difference estimations:");
ToleranceStatistics statistics = _statistics;
log.code(() -> {
// log.info(String.format("Component: %s\nInputs: %s\noutput=%s", component, Arrays.toString(inputPrototype), outputPrototype));
BatchDerivativeTester.log.info(String.format("Finite-Difference Derivative Accuracy:"));
BatchDerivativeTester.log.info(String.format("absoluteTol: %s", statistics.absoluteTol));
BatchDerivativeTester.log.info(String.format("relativeTol: %s", statistics.relativeTol));
});
log.h2("Frozen and Alive Status");
log.code(() -> {
testFrozen(component, ioPair.getInputPrototype());
testUnFrozen(component, ioPair.getInputPrototype());
});
return _statistics;
}
use of com.simiacryptus.mindseye.lang.Tensor in project MindsEye by SimiaCryptus.
the class SumMetaLayer method eval.
@Nullable
@Override
public Result eval(@Nonnull final Result... inObj) {
final Result input = inObj[0];
Arrays.stream(inObj).forEach(nnResult -> nnResult.addRef());
final int itemCnt = input.getData().length();
if (null == lastResult || minBatches < itemCnt) {
@Nonnull final ToDoubleFunction<Coordinate> f = (c) -> IntStream.range(0, itemCnt).mapToDouble(dataIndex -> input.getData().get(dataIndex).get(c)).sum();
lastResult = input.getData().get(0).mapCoords(f);
}
return new Result(TensorArray.wrap(lastResult), (@Nonnull final DeltaSet<Layer> buffer, @Nonnull final TensorList data) -> {
if (input.isAlive()) {
@Nullable final Tensor delta = data.get(0);
@Nonnull final Tensor[] feedback = new Tensor[itemCnt];
Arrays.parallelSetAll(feedback, i -> new Tensor(delta.getDimensions()));
@Nonnull final ToDoubleFunction<Coordinate> f = (inputCoord) -> {
for (int inputItem = 0; inputItem < itemCnt; inputItem++) {
feedback[inputItem].add(inputCoord, delta.get(inputCoord));
}
return 0;
};
delta.mapCoords(f);
@Nonnull TensorArray tensorArray = TensorArray.wrap(feedback);
input.accumulate(buffer, tensorArray);
}
}) {
@Override
protected void _free() {
Arrays.stream(inObj).forEach(nnResult -> nnResult.freeRef());
}
@Override
public boolean isAlive() {
return input.isAlive();
}
};
}
Aggregations