Examples with Pair org.nd4j.linalg.primitives.Pair used on opensource projects

Example 61 with Pair

use of org.nd4j.linalg.primitives.Pair in project nd4j by deeplearning4j.

the class ActivationSoftPlus method backprop.

@Override
public Pair<INDArray, INDArray> backprop(INDArray in, INDArray epsilon) {
    INDArray dLdz = Nd4j.getExecutioner().execAndReturn(new Sigmoid(in));
    dLdz.muli(epsilon);
    return new Pair<>(dLdz, null);
}

Example 62 with Pair

use of org.nd4j.linalg.primitives.Pair in project nd4j by deeplearning4j.

the class DataSetUtil method mergeTimeSeries.

/**
 * Merge the specified time series (3d) arrays and masks. See {@link #mergeFeatures(INDArray[], INDArray[])}
 * and {@link #mergeLabels(INDArray[], INDArray[])}
 *
 * @param arrays   Arrays to merge
 * @param masks    Mask arrays to merge
 * @return Merged arrays and mask
 */
public static Pair<INDArray, INDArray> mergeTimeSeries(INDArray[] arrays, INDArray[] masks) {
    // Merge time series data, and handle masking etc for different length arrays
    // Complications with time series:
    // (a) They may have different lengths (if so: need input + output masking arrays)
    // (b) Even if they are all the same length, they may have masking arrays (if so: merge the masking arrays too)
    // (c) Furthermore: mask arrays can be per-time-step (2d) or per output (3d). Per-input masks (3d feature masks)
    // are not supported, however
    int firstLength = arrays[0].size(2);
    int size = arrays[0].size(1);
    int maxLength = firstLength;
    boolean hasMask = false;
    int maskRank = -1;
    boolean lengthsDiffer = false;
    int totalExamples = 0;
    for (int i = 0; i < arrays.length; i++) {
        totalExamples += arrays[i].size(0);
        int thisLength = arrays[i].size(2);
        maxLength = Math.max(maxLength, thisLength);
        if (thisLength != firstLength)
            lengthsDiffer = true;
        if (masks != null && masks[i] != null && masks[i] != null) {
            maskRank = masks[i].rank();
            hasMask = true;
        }
        if (arrays[i].size(1) != size) {
            throw new IllegalStateException("Cannot merge time series with different size for dimension 1 (first shape: " + Arrays.toString(arrays[0].shape()) + ", " + i + "th shape: " + Arrays.toString(arrays[i].shape()));
        }
    }
    boolean needMask = hasMask || lengthsDiffer;
    INDArray arr = Nd4j.create(totalExamples, size, maxLength);
    INDArray mask = (needMask && maskRank != 3 ? Nd4j.ones(totalExamples, maxLength) : null);
    // Now, merge the time series (and if necessary, mask arrays):
    int examplesSoFar = 0;
    if (!lengthsDiffer && !needMask) {
        // Simplest case: same length, no mask arrays
        for (int i = 0; i < arrays.length; i++) {
            int thisNExamples = arrays[i].size(0);
            arr.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.all(), NDArrayIndex.all() }, arrays[i]);
            examplesSoFar += thisNExamples;
        }
        return new Pair<>(arr, null);
    } else {
        // Either different length, or have mask arrays (or, both)
        if ((lengthsDiffer && !hasMask) || maskRank == 2) {
            // Standard per-example masking required
            for (int i = 0; i < arrays.length; i++) {
                INDArray a = arrays[i];
                int thisNExamples = a.size(0);
                int thisLength = a.size(2);
                arr.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.all(), NDArrayIndex.interval(0, thisLength) }, a);
                if (masks != null && masks[i] != null && masks[i] != null) {
                    INDArray origMask = masks[i];
                    int maskLength = origMask.size(1);
                    mask.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.interval(0, maskLength) }, origMask);
                    if (maskLength < maxLength) {
                        // Set end mask array to zero...
                        mask.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.interval(maskLength, maxLength) }, Nd4j.zeros(thisNExamples, maxLength - maskLength));
                    }
                } else {
                    if (thisLength < maxLength) {
                        // Mask the end
                        mask.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.interval(thisLength, maxLength) }, Nd4j.zeros(thisNExamples, maxLength - thisLength));
                    }
                }
                examplesSoFar += thisNExamples;
            }
        } else if (maskRank == 3) {
            // Per output masking required. May also be variable length
            mask = Nd4j.create(arr.shape());
            for (int i = 0; i < arrays.length; i++) {
                INDArray m = masks[i];
                INDArray a = arrays[i];
                int thisNExamples = a.size(0);
                int thisLength = a.size(2);
                arr.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.all(), NDArrayIndex.interval(0, thisLength) }, a);
                if (m == null) {
                    // This mask is null -> equivalent to "all present"
                    mask.get(NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.all(), NDArrayIndex.interval(0, thisLength)).assign(1);
                } else {
                    mask.put(new INDArrayIndex[] { NDArrayIndex.interval(examplesSoFar, examplesSoFar + thisNExamples), NDArrayIndex.all(), NDArrayIndex.interval(0, thisLength) }, m);
                }
                examplesSoFar += thisNExamples;
            }
        } else {
            throw new UnsupportedOperationException("Cannot merge time series with mask rank " + maskRank);
        }
    }
    return new Pair<>(arr, mask);
}

Example 63 with Pair

use of org.nd4j.linalg.primitives.Pair in project nd4j by deeplearning4j.

the class DataSetUtil method merge4d.

/**
 * Merge the specified 4d arrays and masks. See {@link #mergeFeatures(INDArray[], INDArray[])}
 * and {@link #mergeLabels(INDArray[], INDArray[])}
 *
 * @param arrays   Arrays to merge
 * @param masks    Mask arrays to merge
 * @return Merged arrays and mask
 */
public static Pair<INDArray, INDArray> merge4d(INDArray[] arrays, INDArray[] masks) {
    // 4d -> images. In principle: could have 2d mask arrays (per-example masks)
    int nExamples = 0;
    int[] shape = arrays[0].shape();
    INDArray[] temp = new INDArray[arrays.length];
    boolean hasMasks = false;
    for (int i = 0; i < arrays.length; i++) {
        nExamples += arrays[i].size(0);
        int[] thisShape = arrays[i].shape();
        if (thisShape.length != 4) {
            throw new IllegalStateException("Cannot merge 4d arrays with non 4d arrays");
        }
        for (int j = 1; j < 4; j++) {
            if (thisShape[j] != shape[j])
                throw new IllegalStateException("Cannot merge 4d arrays with different shape (other than # examples): " + " data[0].shape = " + Arrays.toString(shape) + ", data[" + i + "].shape = " + Arrays.toString(thisShape));
        }
        temp[i] = arrays[i];
        if (masks != null && masks[i] != null && masks[i] != null) {
            hasMasks = true;
            if (masks[i].rank() != 2) {
                throw new UnsupportedOperationException("Cannot merged 4d arrays with masks that are not rank 2." + " Got mask array with rank: " + masks[i].rank());
            }
        }
    }
    INDArray out = Nd4j.specialConcat(0, temp);
    INDArray outMask = null;
    if (hasMasks) {
        outMask = DataSetUtil.mergePerOutputMasks2d(out.shape(), arrays, masks);
    }
    return new Pair<>(out, outMask);
}

Example 64 with Pair

use of org.nd4j.linalg.primitives.Pair in project nd4j by deeplearning4j.

the class Nd4jTest method testExpandDims.

@Test
public void testExpandDims() {
    final List<Pair<INDArray, String>> testMatricesC = NDArrayCreationUtil.getAllTestMatricesWithShape('c', 3, 5, 0xDEAD);
    final List<Pair<INDArray, String>> testMatricesF = NDArrayCreationUtil.getAllTestMatricesWithShape('f', 7, 11, 0xBEEF);
    final ArrayList<Pair<INDArray, String>> testMatrices = new ArrayList<>(testMatricesC);
    testMatrices.addAll(testMatricesF);
    for (Pair<INDArray, String> testMatrixPair : testMatrices) {
        final String recreation = testMatrixPair.getSecond();
        final INDArray testMatrix = testMatrixPair.getFirst();
        final char ordering = testMatrix.ordering();
        final int[] shape = testMatrix.shape();
        final int rank = testMatrix.rank();
        for (int i = -rank; i <= rank; i++) {
            final INDArray expanded = Nd4j.expandDims(testMatrix, i);
            final String message = "Expanding in Dimension " + i + "; Shape before expanding: " + Arrays.toString(shape) + " " + ordering + " Order; Shape after expanding: " + Arrays.toString(expanded.shape()) + " " + expanded.ordering() + "; Input Created via: " + recreation;
            assertEquals(message, 1, expanded.shape()[i < 0 ? i + rank : i]);
            assertEquals(message, testMatrix.ravel(), expanded.ravel());
            assertEquals(message, ordering, expanded.ordering());
            testMatrix.assign(Nd4j.rand(shape));
            assertEquals(message, testMatrix.ravel(), expanded.ravel());
        }
    }
}

Example 65 with Pair

use of org.nd4j.linalg.primitives.Pair in project nd4j by deeplearning4j.

the class StaticShapeTests method testBufferToIntShapeStrideMethods.

@Test
public void testBufferToIntShapeStrideMethods() {
    // Specifically: Shape.shape(IntBuffer), Shape.shape(DataBuffer)
    // .isRowVectorShape(DataBuffer), .isRowVectorShape(IntBuffer)
    // Shape.size(DataBuffer,int), Shape.size(IntBuffer,int)
    // Also: Shape.stride(IntBuffer), Shape.stride(DataBuffer)
    // Shape.stride(DataBuffer,int), Shape.stride(IntBuffer,int)
    List<List<Pair<INDArray, String>>> lists = new ArrayList<>();
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 4, 12345));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(1, 4, 12345));
    lists.add(NDArrayCreationUtil.getAllTestMatricesWithShape(3, 1, 12345));
    lists.add(NDArrayCreationUtil.getAll3dTestArraysWithShape(12345, 3, 4, 5));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, 3, 4, 5, 6));
    lists.add(NDArrayCreationUtil.getAll4dTestArraysWithShape(12345, 3, 1, 5, 1));
    lists.add(NDArrayCreationUtil.getAll5dTestArraysWithShape(12345, 3, 4, 5, 6, 7));
    lists.add(NDArrayCreationUtil.getAll6dTestArraysWithShape(12345, 3, 4, 5, 6, 7, 8));
    int[][] shapes = new int[][] { { 3, 4 }, { 1, 4 }, { 3, 1 }, { 3, 4, 5 }, { 3, 4, 5, 6 }, { 3, 1, 5, 1 }, { 3, 4, 5, 6, 7 }, { 3, 4, 5, 6, 7, 8 } };
    for (int i = 0; i < shapes.length; i++) {
        List<Pair<INDArray, String>> list = lists.get(i);
        int[] shape = shapes[i];
        for (Pair<INDArray, String> p : list) {
            INDArray arr = p.getFirst();
            assertArrayEquals(shape, arr.shape());
            int[] thisStride = arr.stride();
            IntBuffer ib = arr.shapeInfo();
            DataBuffer db = arr.shapeInfoDataBuffer();
            // Check shape calculation
            assertEquals(shape.length, Shape.rank(ib));
            assertEquals(shape.length, Shape.rank(db));
            assertArrayEquals(shape, Shape.shape(ib));
            assertArrayEquals(shape, Shape.shape(db));
            for (int j = 0; j < shape.length; j++) {
                assertEquals(shape[j], Shape.size(ib, j));
                assertEquals(shape[j], Shape.size(db, j));
                assertEquals(thisStride[j], Shape.stride(ib, j));
                assertEquals(thisStride[j], Shape.stride(db, j));
            }
            // Check base offset
            assertEquals(Shape.offset(ib), Shape.offset(db));
            // Check offset calculation:
            NdIndexIterator iter = new NdIndexIterator(shape);
            while (iter.hasNext()) {
                int[] next = iter.next();
                long offset1 = Shape.getOffset(ib, next);
                assertEquals(offset1, Shape.getOffset(db, next));
                switch(shape.length) {
                    case 2:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1]));
                        break;
                    case 3:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2]));
                        break;
                    case 4:
                        assertEquals(offset1, Shape.getOffset(ib, next[0], next[1], next[2], next[3]));
                        assertEquals(offset1, Shape.getOffset(db, next[0], next[1], next[2], next[3]));
                        break;
                    case 5:
                    case 6:
                        // No 5 and 6d getOffset overloads
                        break;
                    default:
                        throw new RuntimeException();
                }
            }
        }
    }
}