Examples with ToleranceStatistics com.simiacryptus.mindseye.test.ToleranceStatistics used on opensource projects

Example 16 with ToleranceStatistics

use of com.simiacryptus.mindseye.test.ToleranceStatistics in project MindsEye by SimiaCryptus.

the class CudaLayerTester method testNonstandardBounds.

/**
 * Test nonstandard bounds tolerance statistics.
 *
 * @param log            the log
 * @param reference      the reference
 * @param inputPrototype the input prototype
 * @return the tolerance statistics
 */
@Nonnull
public ToleranceStatistics testNonstandardBounds(final NotebookOutput log, @Nullable final Layer reference, @Nonnull final Tensor[] inputPrototype) {
    log.h2("Irregular Input");
    log.p("This layer should be able to accept non-dense inputs.");
    return log.code(() -> {
        Tensor[] randomized = Arrays.stream(inputPrototype).map(x -> x.map(v -> getRandom())).toArray(i -> new Tensor[i]);
        logger.info("Input: " + Arrays.stream(randomized).map(Tensor::prettyPrint).collect(Collectors.toList()));
        Precision precision = Precision.Double;
        TensorList[] controlInput = CudaSystem.run(gpu -> {
            return Arrays.stream(randomized).map(original -> {
                TensorArray data = TensorArray.create(original);
                CudaTensorList wrap = CudaTensorList.wrap(gpu.getTensor(data, precision, MemoryType.Managed, false), 1, original.getDimensions(), precision);
                data.freeRef();
                return wrap;
            }).toArray(i -> new TensorList[i]);
        }, 0);
        @Nonnull final SimpleResult controlResult = CudaSystem.run(gpu -> {
            return SimpleGpuEval.run(reference, gpu, controlInput);
        }, 1);
        final TensorList[] irregularInput = CudaSystem.run(gpu -> {
            return Arrays.stream(randomized).map(original -> {
                return buildIrregularCudaTensor(gpu, precision, original);
            }).toArray(i -> new TensorList[i]);
        }, 0);
        @Nonnull final SimpleResult testResult = CudaSystem.run(gpu -> {
            return SimpleGpuEval.run(reference, gpu, irregularInput);
        }, 1);
        try {
            ToleranceStatistics compareOutput = compareOutput(controlResult, testResult);
            ToleranceStatistics compareDerivatives = compareDerivatives(controlResult, testResult);
            return compareDerivatives.combine(compareOutput);
        } finally {
            Arrays.stream(randomized).forEach(ReferenceCountingBase::freeRef);
            Arrays.stream(controlInput).forEach(ReferenceCounting::freeRef);
            Arrays.stream(irregularInput).forEach(x -> x.freeRef());
            controlResult.freeRef();
            testResult.freeRef();
        }
    });
}

Example 17 with ToleranceStatistics

use of com.simiacryptus.mindseye.test.ToleranceStatistics in project MindsEye by SimiaCryptus.

the class CudaLayerTester method compareDerivatives.

/**
 * Compare derivatives tolerance statistics.
 *
 * @param expected the expected
 * @param actual   the actual
 * @return the tolerance statistics
 */
@Nonnull
public ToleranceStatistics compareDerivatives(final SimpleResult expected, final SimpleResult actual) {
    ToleranceStatistics derivativeAgreement = compareInputDerivatives(expected, actual);
    derivativeAgreement = derivativeAgreement.combine(compareLayerDerivatives(expected, actual));
    return derivativeAgreement;
}

Example 18 with ToleranceStatistics

use of com.simiacryptus.mindseye.test.ToleranceStatistics in project MindsEye by SimiaCryptus.

the class ReferenceIO method test.

@Nullable
@Override
public ToleranceStatistics test(@Nonnull final NotebookOutput log, @Nonnull final Layer layer, @Nonnull final Tensor... inputPrototype) {
    if (!referenceIO.isEmpty()) {
        log.h1("Reference Input/Output Pairs");
        log.p("Display pre-setBytes input/output example pairs:");
        referenceIO.forEach((input, output) -> {
            log.code(() -> {
                @Nonnull final SimpleEval eval = SimpleEval.run(layer, input);
                Tensor add = output.scale(-1).addAndFree(eval.getOutput());
                @Nonnull final DoubleStatistics error = new DoubleStatistics().accept(add.getData());
                add.freeRef();
                String format = String.format("--------------------\nInput: \n[%s]\n--------------------\nOutput: \n%s\nError: %s\n--------------------\nDerivative: \n%s", Arrays.stream(input).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).get(), eval.getOutput().prettyPrint(), error, Arrays.stream(eval.getDerivative()).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).get());
                eval.freeRef();
                return format;
            });
        });
    } else {
        log.h1("Example Input/Output Pair");
        log.p("Display input/output pairs from random executions:");
        log.code(() -> {
            @Nonnull final SimpleEval eval = SimpleEval.run(layer, inputPrototype);
            String format = String.format("--------------------\nInput: \n[%s]\n--------------------\nOutput: \n%s\n--------------------\nDerivative: \n%s", Arrays.stream(inputPrototype).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).orElse(""), eval.getOutput().prettyPrint(), Arrays.stream(eval.getDerivative()).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).orElse(""));
            eval.freeRef();
            return format;
        });
    }
    return null;
}

Example 19 with ToleranceStatistics

use of com.simiacryptus.mindseye.test.ToleranceStatistics in project MindsEye by SimiaCryptus.

the class SingleDerivativeTester method test.

/**
 * Test tolerance statistics.
 *
 * @param output
 * @param component      the component
 * @param inputPrototype the input prototype
 * @return the tolerance statistics
 */
@Override
public ToleranceStatistics test(@Nonnull final NotebookOutput output, @Nonnull final Layer component, @Nonnull final Tensor... inputPrototype) {
    output.h1("Differential Validation");
    ToleranceStatistics _statistics = new ToleranceStatistics();
    final Tensor outputPrototype = SimpleEval.run(component, inputPrototype).getOutputAndFree();
    try {
        if (verbose) {
            output.code(() -> {
                log.info(String.format("Inputs: %s", Arrays.stream(inputPrototype).map(t -> t.prettyPrint()).reduce((a, b) -> a + ",\n" + b).orElse("")));
                log.info(String.format("Inputs Statistics: %s", Arrays.stream(inputPrototype).map(x -> new ScalarStatistics().add(x.getData()).toString()).reduce((a, b) -> a + ",\n" + b).orElse("")));
                log.info(String.format("Output: %s", null == outputPrototype ? null : outputPrototype.prettyPrint()));
                log.info(String.format("Outputs Statistics: %s", new ScalarStatistics().add(outputPrototype.getData())));
            });
        }
        if (isTestFeedback()) {
            output.h2("Feedback Validation");
            output.p("We validate the agreement between the implemented derivative _of the inputs_ apply finite difference estimations:");
            final ToleranceStatistics statistics = _statistics;
            _statistics = output.code(() -> {
                return testFeedback(statistics, component, inputPrototype, outputPrototype);
            });
        }
        if (isTestLearning()) {
            output.h2("Learning Validation");
            output.p("We validate the agreement between the implemented derivative _of the internal weights_ apply finite difference estimations:");
            final ToleranceStatistics statistics = _statistics;
            _statistics = output.code(() -> {
                return testLearning(statistics, component, inputPrototype, outputPrototype);
            });
        }
    } finally {
        outputPrototype.freeRef();
    }
    output.h2("Total Accuracy");
    output.p("The overall agreement accuracy between the implemented derivative and the finite difference estimations:");
    final ToleranceStatistics statistics = _statistics;
    output.code(() -> {
        // log.info(String.format("Component: %s\nInputs: %s\noutput=%s", component, Arrays.toString(inputPrototype), outputPrototype));
        log.info(String.format("Finite-Difference Derivative Accuracy:"));
        log.info(String.format("absoluteTol: %s", statistics.absoluteTol));
        log.info(String.format("relativeTol: %s", statistics.relativeTol));
    });
    output.h2("Frozen and Alive Status");
    output.code(() -> {
        testFrozen(component, inputPrototype);
        testUnFrozen(component, inputPrototype);
    });
    return _statistics;
}

Example 20 with ToleranceStatistics

use of com.simiacryptus.mindseye.test.ToleranceStatistics in project MindsEye by SimiaCryptus.

the class CudaLayerTestBase method getReferenceIOTester.

@Nullable
@Override
protected ComponentTest<ToleranceStatistics> getReferenceIOTester() {
    @Nullable final ComponentTest<ToleranceStatistics> inner = super.getReferenceIOTester();
    return new ComponentTestBase<ToleranceStatistics>() {

        @Override
        protected void _free() {
            inner.freeRef();
            super._free();
        }

        @Override
        public ToleranceStatistics test(@Nonnull NotebookOutput log, Layer component, Tensor... inputPrototype) {
            @Nullable PrintStream apiLog = null;
            try {
                @Nonnull String logName = "cuda_" + log.getName() + "_io.log";
                log.p(log.file((String) null, logName, "GPU Log"));
                apiLog = new PrintStream(log.file(logName));
                CudaSystem.addLog(apiLog);
                return inner.test(log, component, inputPrototype);
            } finally {
                if (null != apiLog) {
                    apiLog.close();
                    CudaSystem.apiLog.remove(apiLog);
                }
            }
        }
    };
}