Examples with DataBuffer org.nd4j.linalg.api.buffer.DataBuffer used on opensource projects

Example 81 with DataBuffer

use of org.nd4j.linalg.api.buffer.DataBuffer in project nd4j by deeplearning4j.

the class BasicTADManagerTest method testTADcreation2.

@Test
public void testTADcreation2() throws Exception {
    INDArray array = Nd4j.create(10, 100);
    TADManager tadManager = new DeviceTADManager();
    DataBuffer tad = tadManager.getTADOnlyShapeInfo(array, new int[] { 0 }).getFirst();
    DataBuffer tad2 = tadManager.getTADOnlyShapeInfo(array, new int[] { 0 }).getFirst();
    System.out.println("TAD: " + tad);
    System.out.println("Shape: " + array.shapeInfoDataBuffer());
    CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext();
    assertEquals(2, tad.getInt(0));
    assertEquals(1, tad.getInt(1));
    assertEquals(10, tad.getInt(2));
    assertEquals(1, tad.getInt(3));
    assertEquals(100, tad.getInt(4));
    assertEquals(0, tad.getInt(5));
    assertEquals(100, tad.getInt(6));
    assertEquals(99, tad.getInt(7));
    assertFalse(AtomicAllocator.getInstance().getAllocationPoint(tad).isActualOnDeviceSide());
    long tadPointer1 = AtomicAllocator.getInstance().getPointer(tad, context).address();
    long tadPointer2 = AtomicAllocator.getInstance().getPointer(tad2, context).address();
    assertTrue(AtomicAllocator.getInstance().getAllocationPoint(tad).isActualOnDeviceSide());
    System.out.println("tadPointer1: " + tadPointer1);
    System.out.println("tadPointer2: " + tadPointer2);
    assertEquals(tadPointer1, tadPointer2);
    AtomicAllocator.getInstance().moveToConstant(tad);
    long tadPointer3 = AtomicAllocator.getInstance().getPointer(tad, context).address();
    long tadPointer4 = AtomicAllocator.getInstance().getPointer(tad2, context).address();
    assertEquals(tadPointer4, tadPointer3);
    assertNotEquals(tadPointer1, tadPointer3);
}

Example 82 with DataBuffer

use of org.nd4j.linalg.api.buffer.DataBuffer in project nd4j by deeplearning4j.

the class OpProfiler method processTADOperands.

public PenaltyCause[] processTADOperands(DataBuffer... tadBuffers) {
    List<PenaltyCause> causes = new ArrayList<>();
    for (DataBuffer tadBuffer : tadBuffers) {
        if (tadBuffer == null)
            continue;
        int rank = tadBuffer.getInt(0);
        int length = rank * 2 + 4;
        int ews = tadBuffer.getInt(length - 2);
        if ((ews < 1 || rank > 2 || (rank == 2 && tadBuffer.getInt(1) > 1 && tadBuffer.getInt(2) > 1)) && !causes.contains(PenaltyCause.TAD_NON_EWS_ACCESS))
            causes.add(PenaltyCause.TAD_NON_EWS_ACCESS);
        else if (ews > 1 && !causes.contains(PenaltyCause.TAD_STRIDED_ACCESS))
            causes.add(PenaltyCause.TAD_STRIDED_ACCESS);
    }
    if (causes.isEmpty())
        causes.add(NONE);
    return causes.toArray(new PenaltyCause[0]);
}

Example 83 with DataBuffer

use of org.nd4j.linalg.api.buffer.DataBuffer in project nd4j by deeplearning4j.

the class BinarySerde method readShapeFromDisk.

/**
 * This method returns shape databuffer from saved earlier file
 *
 * @param readFrom
 * @return
 * @throws IOException
 */
public static DataBuffer readShapeFromDisk(File readFrom) throws IOException {
    try (FileInputStream os = new FileInputStream(readFrom)) {
        FileChannel channel = os.getChannel();
        // we read shapeinfo up to max_rank value, which is 32
        int len = (int) Math.min((32 * 2 + 3) * 4, readFrom.length());
        ByteBuffer buffer = ByteBuffer.allocateDirect(len);
        channel.read(buffer);
        ByteBuffer byteBuffer = buffer == null ? ByteBuffer.allocateDirect(buffer.array().length).put(buffer.array()).order(ByteOrder.nativeOrder()) : buffer.order(ByteOrder.nativeOrder());
        buffer.position(0);
        int rank = byteBuffer.getInt();
        int[] result = new int[Shape.shapeInfoLength(rank)];
        // filling DataBuffer with shape info
        result[0] = rank;
        // skipping two next values (dtype and rank again)
        byteBuffer.position(12);
        // filling shape information
        for (int e = 1; e < Shape.shapeInfoLength(rank); e++) {
            result[e] = byteBuffer.getInt();
        }
        // creating nd4j databuffer now
        DataBuffer dataBuffer = Nd4j.getDataBufferFactory().createInt(result);
        return dataBuffer;
    }
}

Example 84 with DataBuffer

use of org.nd4j.linalg.api.buffer.DataBuffer in project nd4j by deeplearning4j.

the class BinarySerde method toArrayAndByteBuffer.

/**
 * Create an ndarray and existing bytebuffer
 * @param buffer
 * @param offset
 * @return
 */
public static Pair<INDArray, ByteBuffer> toArrayAndByteBuffer(ByteBuffer buffer, int offset) {
    ByteBuffer byteBuffer = buffer == null ? ByteBuffer.allocateDirect(buffer.array().length).put(buffer.array()).order(ByteOrder.nativeOrder()) : buffer.order(ByteOrder.nativeOrder());
    // bump the byte buffer to the proper position
    byteBuffer.position(offset);
    int rank = byteBuffer.getInt();
    if (rank < 0)
        throw new IllegalStateException("Found negative integer. Corrupt serialization?");
    // get the shape buffer length to create the shape information buffer
    int shapeBufferLength = Shape.shapeInfoLength(rank);
    // create the ndarray shape information
    DataBuffer shapeBuff = Nd4j.createBufferDetached(new int[shapeBufferLength]);
    // compute the databuffer opType from the index
    DataBuffer.Type type = DataBuffer.Type.values()[byteBuffer.getInt()];
    for (int i = 0; i < shapeBufferLength; i++) {
        shapeBuff.put(i, byteBuffer.getInt());
    }
    // after the rank,data opType, shape buffer (of length shape buffer length) * sizeof(int)
    if (type != DataBuffer.Type.COMPRESSED) {
        ByteBuffer slice = byteBuffer.slice();
        // wrap the data buffer for the last bit
        DataBuffer buff = Nd4j.createBuffer(slice, type, Shape.length(shapeBuff));
        // advance past the data
        int position = byteBuffer.position() + (buff.getElementSize() * (int) buff.length());
        byteBuffer.position(position);
        // create the final array
        // TODO: see how to avoid dup here
        INDArray arr = Nd4j.createArrayFromShapeBuffer(buff.dup(), shapeBuff.dup());
        return Pair.of(arr, byteBuffer);
    } else {
        CompressionDescriptor compressionDescriptor = CompressionDescriptor.fromByteBuffer(byteBuffer);
        ByteBuffer slice = byteBuffer.slice();
        // ensure that we only deal with the slice of the buffer that is actually the data
        BytePointer byteBufferPointer = new BytePointer(slice);
        // create a compressed array based on the rest of the data left in the buffer
        CompressedDataBuffer compressedDataBuffer = new CompressedDataBuffer(byteBufferPointer, compressionDescriptor);
        // TODO: see how to avoid dup()
        INDArray arr = Nd4j.createArrayFromShapeBuffer(compressedDataBuffer.dup(), shapeBuff.dup());
        // advance past the data
        int compressLength = (int) compressionDescriptor.getCompressedLength();
        byteBuffer.position(byteBuffer.position() + compressLength);
        return Pair.of(arr, byteBuffer);
    }
}

Example 85 with DataBuffer

use of org.nd4j.linalg.api.buffer.DataBuffer in project nd4j by deeplearning4j.

the class Nd4jTestsC method testVectorInit.

@Test
public void testVectorInit() {
    DataBuffer data = Nd4j.linspace(1, 4, 4).data();
    INDArray arr = Nd4j.create(data, new int[] { 1, 4 });
    assertEquals(true, arr.isRowVector());
    INDArray arr2 = Nd4j.create(data, new int[] { 1, 4 });
    assertEquals(true, arr2.isRowVector());
    INDArray columnVector = Nd4j.create(data, new int[] { 4, 1 });
    assertEquals(true, columnVector.isColumnVector());
}