Examples with DeltaSet com.simiacryptus.mindseye.lang.DeltaSet used on opensource projects

Example 31 with DeltaSet

use of com.simiacryptus.mindseye.lang.DeltaSet in project MindsEye by SimiaCryptus.

the class AvgReducerLayer method evalAndFree.

@Nullable
@Override
public Result evalAndFree(final Result... inObj) {
    if (!CudaSystem.isEnabled())
        return getCompatibilityLayer().evalAndFree(inObj);
    final Result input = inObj[0];
    final TensorList inputData = input.getData();
    @Nonnull final int[] inputSize = inputData.getDimensions();
    int length = inputData.length();
    CudaTensorList result = CudaSystem.run(gpu -> {
        CudaTensor inputTensor = gpu.getTensor(inputData, precision, MemoryType.Device, false);
        inputData.freeRef();
        CudaMemory inputMemory = inputTensor.getMemory(gpu);
        @Nonnull final CudaDevice.CudaTensorDescriptor outputDescriptor = gpu.newTensorDescriptor(precision, length, 1, 1, 1);
        long size = (long) precision.size * outputDescriptor.nStride * length;
        @Nonnull final CudaMemory outputMemory = gpu.allocate(size, MemoryType.Managed, true);
        CudaResource<cudnnReduceTensorDescriptor> reduceTensorDescriptor = gpu.cudnnCreateReduceTensorDescriptor(cudnnReduceTensorOp.CUDNN_REDUCE_TENSOR_AVG, precision.code, cudnnNanPropagation.CUDNN_NOT_PROPAGATE_NAN, cudnnReduceTensorIndices.CUDNN_REDUCE_TENSOR_NO_INDICES, cudnnIndicesType.CUDNN_32BIT_INDICES);
        @Nonnull final CudaMemory workspacePtr = gpu.allocate(inputMemory.size, MemoryType.Device, true);
        @Nonnull final CudaMemory indexPtr = gpu.allocate(12 * length, MemoryType.Device, false);
        // outputPtr.synchronize();
        gpu.cudnnReduceTensor(reduceTensorDescriptor.getPtr(), indexPtr.getPtr(), indexPtr.size, workspacePtr.getPtr(), workspacePtr.size, precision.getPointer(1.0), inputTensor.descriptor.getPtr(), inputMemory.getPtr(), precision.getPointer(0.0), outputDescriptor.getPtr(), outputMemory.getPtr());
        outputMemory.dirty();
        inputMemory.dirty();
        Stream.of(inputTensor, inputMemory, reduceTensorDescriptor, workspacePtr, indexPtr).forEach(ReferenceCounting::freeRef);
        return CudaTensorList.wrap(CudaTensor.wrap(outputMemory, outputDescriptor, precision), length, new int[] { 1, 1, 1 }, precision);
    });
    return new Result(result, (DeltaSet<Layer> ctx, TensorList delta) -> {
        // Not supported by CuDNN?
        // CudaTensorList passback = CudaSystem.run(gpu -> {
        // CudaTensor deltaTensor = gpu.getTensor(delta, precision, MemoryType.Device, false);
        // CudaMemory deltaMemory = deltaTensor.getMemory(gpu);
        // 
        // @Nonnull final CudaDevice.CudaTensorDescriptor passbackDescriptor1 = gpu.newTensorDescriptor(
        // precision, length, inputSize[2], inputSize[1], inputSize[0]
        // );
        // @Nonnull final CudaMemory passbackPtr1 = gpu.allocate((long) precision.size * passbackDescriptor1.nStride * length, MemoryType.Device, false);
        // gpu.cudnnAddTensor(precision.getPointer(1.0), deltaTensor.descriptor.getPtr(), deltaMemory.getPtr(),
        // precision.getPointer(1.0), passbackDescriptor1.getPtr(), passbackPtr1.getPtr());
        // passbackPtr1.dirty();
        // 
        // Stream.of(deltaTensor, deltaMemory, passbackDescriptor1, passbackPtr1).forEach(ReferenceCounting::freeRef);
        // return CudaTensorList.wrap(CudaTensor.wrap(passbackPtr1, passbackDescriptor1, precision), length, inputSize, precision);
        // });
        TensorList passback = TensorArray.wrap(IntStream.range(0, length).mapToObj(i -> {
            Tensor tensor = delta.get(i);
            Tensor tensor1 = new Tensor(inputSize).setAll((double) tensor.get(0) / Tensor.length(inputSize));
            tensor.freeRef();
            return tensor1;
        }).toArray(i -> new Tensor[i]));
        input.accumulate(ctx, passback);
    }) {

        @Override
        protected void _free() {
            super._free();
            input.freeRef();
        }
    };
}

Example 32 with DeltaSet

use of com.simiacryptus.mindseye.lang.DeltaSet in project MindsEye by SimiaCryptus.

the class ConvolutionLayer method evalAndFree.

@Nullable
@Override
public Result evalAndFree(@Nonnull final Result... inObj) {
    final Tensor kernel = getKernel();
    kernel.addRef();
    assert kernel.isValid();
    assert 1 == inObj.length;
    assert 3 == inObj[0].getData().getDimensions().length;
    assert inputBands == inObj[0].getData().getDimensions()[2] : Arrays.toString(inObj[0].getData().getDimensions()) + "[2] != " + inputBands;
    if (!CudaSystem.isEnabled())
        return getCompatibilityLayer().evalAndFree(inObj);
    @Nonnull ExplodedConvolutionGrid grid = getExplodedNetwork();
    @Nonnull PipelineNetwork network = grid.getNetwork();
    if (isFrozen()) {
        network.freeze();
    }
    final Result result = network.evalAndFree(inObj);
    network.freeRef();
    final TensorList resultData = result.getData();
    assert inObj[0].getData().length() == resultData.length();
    assert 3 == resultData.getDimensions().length;
    assert outputBands == resultData.getDimensions()[2];
    ConvolutionLayer.this.addRef();
    return new Result(resultData, (@Nonnull final DeltaSet<Layer> deltaSet, @Nonnull final TensorList delta) -> {
        result.accumulate(deltaSet, delta);
        if (!isFrozen()) {
            Tensor read = grid.read(deltaSet, true);
            deltaSet.get(ConvolutionLayer.this, kernel.getData()).addInPlace(read.getData()).freeRef();
            read.freeRef();
        }
    }) {

        @Override
        public void accumulate(final DeltaSet<Layer> buffer, final TensorList delta) {
            getAccumulator().accept(buffer, delta);
        }

        @Override
        protected void _free() {
            grid.freeRef();
            result.freeRef();
            kernel.freeRef();
            ConvolutionLayer.this.freeRef();
        }

        @Override
        public boolean isAlive() {
            return result.isAlive();
        }
    };
}

Example 33 with DeltaSet

use of com.simiacryptus.mindseye.lang.DeltaSet in project MindsEye by SimiaCryptus.

the class BasicTrainable method eval.

/**
 * Eval point sample.
 *
 * @param list    the list
 * @param monitor the monitor
 * @return the point sample
 */
@Nonnull
protected PointSample eval(@Nonnull final List<Tensor[]> list, @Nullable final TrainingMonitor monitor) {
    @Nonnull final TimedResult<PointSample> timedResult = TimedResult.time(() -> {
        final Result[] nnContext = BasicTrainable.getNNContext(list, mask);
        final Result result = network.eval(nnContext);
        for (@Nonnull Result nnResult : nnContext) {
            nnResult.getData().freeRef();
            nnResult.freeRef();
        }
        final TensorList resultData = result.getData();
        @Nonnull final DeltaSet<Layer> deltaSet = new DeltaSet<Layer>();
        @Nonnull StateSet<Layer> stateSet = null;
        try {
            final DoubleSummaryStatistics statistics = resultData.stream().flatMapToDouble(x -> {
                double[] array = Arrays.stream(x.getData()).toArray();
                x.freeRef();
                return Arrays.stream(array);
            }).summaryStatistics();
            final double sum = statistics.getSum();
            result.accumulate(deltaSet, 1.0);
            stateSet = new StateSet<>(deltaSet);
            // log.info(String.format("Evaluated to %s delta buffers, %s mag", DeltaSet<LayerBase>.getMap().size(), DeltaSet<LayerBase>.getMagnitude()));
            return new PointSample(deltaSet, stateSet, sum, 0.0, list.size());
        } finally {
            if (null != stateSet)
                stateSet.freeRef();
            resultData.freeRefAsync();
            result.freeRefAsync();
            deltaSet.freeRefAsync();
        }
    });
    if (null != monitor && verbosity() > 0) {
        monitor.log(String.format("Device completed %s items in %.3f sec", list.size(), timedResult.timeNanos / 1e9));
    }
    @Nonnull PointSample normalize = timedResult.result.normalize();
    timedResult.result.freeRef();
    return normalize;
}

Example 34 with DeltaSet

use of com.simiacryptus.mindseye.lang.DeltaSet in project MindsEye by SimiaCryptus.

the class L12Normalizer method measure.

@Nonnull
@Override
public PointSample measure(final TrainingMonitor monitor) {
    final PointSample innerMeasure = inner.measure(monitor);
    @Nonnull final DeltaSet<Layer> normalizationVector = new DeltaSet<Layer>();
    double valueAdj = 0;
    for (@Nonnull final Layer layer : getLayers(innerMeasure.delta.getMap().keySet())) {
        final double[] weights = innerMeasure.delta.getMap().get(layer).target;
        @Nullable final double[] gradientAdj = normalizationVector.get(layer, weights).getDelta();
        final double factor_L1 = getL1(layer);
        final double factor_L2 = getL2(layer);
        assert null != gradientAdj;
        for (int i = 0; i < gradientAdj.length; i++) {
            final double sign = weights[i] < 0 ? -1.0 : 1.0;
            gradientAdj[i] += factor_L1 * sign + 2 * factor_L2 * weights[i];
            valueAdj += (factor_L1 * sign + factor_L2 * weights[i]) * weights[i];
        }
    }
    return new PointSample(innerMeasure.delta.add(normalizationVector), innerMeasure.weights, innerMeasure.sum + (hideAdj ? 0 : valueAdj), innerMeasure.rate, innerMeasure.count).normalize();
}

Example 35 with DeltaSet

use of com.simiacryptus.mindseye.lang.DeltaSet in project MindsEye by SimiaCryptus.

the class SparkTrainable method getDelta.

/**
 * Gets delta.
 *
 * @param reduce the reduce
 * @return the delta
 */
@Nonnull
protected DeltaSet<Layer> getDelta(@Nonnull final SparkTrainable.ReducableResult reduce) {
    @Nonnull final DeltaSet<Layer> xxx = new DeltaSet<Layer>();
    final Tensor[] prototype = dataRDD.toJavaRDD().take(1).get(0);
    final Result result = network.eval(ConstantResult.batchResultArray(new Tensor[][] { prototype }));
    result.accumulate(xxx, 0);
    reduce.accumulate(xxx);
    return xxx;
}