Examples with CompressionDescriptor org.nd4j.linalg.compression.CompressionDescriptor used on opensource projects

Example 16 with CompressionDescriptor

use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.

the class Float8 method compressPointer.

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());
    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT8, ptr, length);
    return buffer;
}

Example 17 with CompressionDescriptor

use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.

the class Int16 method compressPointer.

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    BytePointer ptr = new BytePointer(length * 2);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 2);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());
    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT16, ptr, length);
    return buffer;
}

Example 18 with CompressionDescriptor

use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.

the class BinarySerde method doByteBufferPutCompressed.

/**
 * Setup the given byte buffer
 * for serialization (note that this is for compressed INDArrays)
 * 4 bytes for rank
 * 4 bytes for data opType
 * shape information
 * codec information
 * data opType
 *
 * @param arr the array to setup
 * @param allocated the byte buffer to setup
 * @param rewind whether to rewind the byte buffer or not
 */
public static void doByteBufferPutCompressed(INDArray arr, ByteBuffer allocated, boolean rewind) {
    CompressedDataBuffer compressedDataBuffer = (CompressedDataBuffer) arr.data();
    CompressionDescriptor descriptor = compressedDataBuffer.getCompressionDescriptor();
    ByteBuffer codecByteBuffer = descriptor.toByteBuffer();
    ByteBuffer buffer = arr.data().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    ByteBuffer shapeBuffer = arr.shapeInfoDataBuffer().pointer().asByteBuffer().order(ByteOrder.nativeOrder());
    allocated.putInt(arr.rank());
    // put data opType next so its self describing
    allocated.putInt(arr.data().dataType().ordinal());
    // put shape next
    allocated.put(shapeBuffer);
    // put codec information next
    allocated.put(codecByteBuffer);
    // finally put the data
    allocated.put(buffer);
    if (rewind)
        allocated.rewind();
}