Examples with NDArrayMessage org.nd4j.aeron.ipc.NDArrayMessage used on opensource projects

Example 1 with NDArrayMessage

use of org.nd4j.aeron.ipc.NDArrayMessage in project nd4j by deeplearning4j.

the class InMemoryChunkAccumulator method reassemble.

/**
 * Reassemble an ndarray message
 * from a set of chunks
 *
 * Note that once reassemble is called,
 * the associated chunk lists will automatically
 * be removed from storage.
 *
 * @param id the id to reassemble
 * @return the reassembled message
 */
@Override
public NDArrayMessage reassemble(String id) {
    List<NDArrayMessageChunk> chunkList = chunks.get(id);
    if (chunkList.size() != chunkList.get(0).getNumChunks())
        throw new IllegalStateException("Unable to reassemble message chunk " + id + " missing " + (chunkList.get(0).getNumChunks() - chunkList.size()) + "chunks");
    // ensure the chunks are in contiguous ordering according to their chunk index
    NDArrayMessageChunk[] inOrder = new NDArrayMessageChunk[chunkList.size()];
    for (NDArrayMessageChunk chunk : chunkList) {
        inOrder[chunk.getChunkIndex()] = chunk;
    }
    // reassemble the in order chunks
    NDArrayMessage message = NDArrayMessage.fromChunks(inOrder);
    chunkList.clear();
    chunks.remove(id);
    return message;
}

Example 2 with NDArrayMessage

use of org.nd4j.aeron.ipc.NDArrayMessage in project nd4j by deeplearning4j.

the class NDArrayMessageChunkTests method testChunkSerialization.

@Test
public void testChunkSerialization() {
    NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.ones(1000));
    int chunkSize = 128;
    int numChunks = NDArrayMessage.numChunksForMessage(message, chunkSize);
    NDArrayMessageChunk[] chunks = NDArrayMessage.chunks(message, chunkSize);
    assertEquals(numChunks, chunks.length);
    for (int i = 1; i < numChunks; i++) {
        assertEquals(chunks[0].getMessageType(), chunks[i].getMessageType());
        assertEquals(chunks[0].getId(), chunks[i].getId());
        assertEquals(chunks[0].getChunkSize(), chunks[i].getChunkSize());
        assertEquals(chunks[0].getNumChunks(), chunks[i].getNumChunks());
    }
    ByteBuffer[] concat = new ByteBuffer[chunks.length];
    for (int i = 0; i < concat.length; i++) concat[i] = chunks[i].getData();
    DirectBuffer buffer = NDArrayMessage.toBuffer(message);
    // test equality of direct byte buffer contents vs chunked
    ByteBuffer byteBuffer = buffer.byteBuffer();
    ByteBuffer concatAll = BufferUtil.concat(concat, buffer.capacity());
    byte[] arrays = new byte[byteBuffer.capacity()];
    byteBuffer.rewind();
    byteBuffer.get(arrays);
    byte[] arrays2 = new byte[concatAll.capacity()];
    concatAll.rewind();
    concatAll.get(arrays2);
    assertArrayEquals(arrays, arrays2);
    NDArrayMessage message1 = NDArrayMessage.fromChunks(chunks);
    assertEquals(message, message1);
}

Example 3 with NDArrayMessage

use of org.nd4j.aeron.ipc.NDArrayMessage in project nd4j by deeplearning4j.

the class ChunkAccumulatorTests method testAccumulator.

@Test
public void testAccumulator() {
    ChunkAccumulator chunkAccumulator = new InMemoryChunkAccumulator();
    NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.ones(1000));
    int chunkSize = 128;
    NDArrayMessageChunk[] chunks = NDArrayMessage.chunks(message, chunkSize);
    for (int i = 0; i < chunks.length; i++) {
        chunkAccumulator.accumulateChunk(chunks[i]);
    }
    NDArrayMessage message1 = chunkAccumulator.reassemble(chunks[0].getId());
    assertEquals(message, message1);
}

Example 4 with NDArrayMessage

use of org.nd4j.aeron.ipc.NDArrayMessage in project nd4j by deeplearning4j.

the class UpdaterStorageTests method testNone.

@Test(expected = UnsupportedOperationException.class)
public void testNone() {
    UpdateStorage updateStorage = new NoUpdateStorage();
    NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.scalar(1.0));
    updateStorage.addUpdate(message);
    assertEquals(1, updateStorage.numUpdates());
    assertEquals(message, updateStorage.getUpdate(0));
    updateStorage.close();
}

Example 5 with NDArrayMessage

use of org.nd4j.aeron.ipc.NDArrayMessage in project nd4j by deeplearning4j.

the class UpdaterStorageTests method testInMemory.

@Test
public void testInMemory() {
    UpdateStorage updateStorage = new InMemoryUpdateStorage();
    NDArrayMessage message = NDArrayMessage.wholeArrayUpdate(Nd4j.scalar(1.0));
    updateStorage.addUpdate(message);
    assertEquals(1, updateStorage.numUpdates());
    assertEquals(message, updateStorage.getUpdate(0));
    updateStorage.clear();
    assertEquals(0, updateStorage.numUpdates());
    updateStorage.close();
}