Search in sources :

Example 1 with CompressedDataBuffer

use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.

the class CudaFlexibleThreshold method compress.

@Override
public DataBuffer compress(DataBuffer buffer) {
    INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[] { 1, (int) buffer.length() }));
    double max = temp.amaxNumber().doubleValue();
    int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();
    long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
    int compressedLength = cntAbs + 3;
    // first 3 elements contain header
    IntPointer pointer = new IntPointer(compressedLength);
    pointer.put(0, cntAbs);
    pointer.put(1, (int) buffer.length());
    // please note, this value will be ovewritten anyway
    pointer.put(2, Float.floatToIntBits(threshold));
    CompressionDescriptor descriptor = new CompressionDescriptor();
    // sizeOf(INT)
    descriptor.setCompressedLength(compressedLength * 4);
    descriptor.setOriginalLength(originalLength);
    descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
    descriptor.setNumberOfElements(buffer.length());
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());
    CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);
    Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());
    Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);
    return cbuff;
}
Also used : CompressionDescriptor(org.nd4j.linalg.compression.CompressionDescriptor) INDArray(org.nd4j.linalg.api.ndarray.INDArray) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer) IntPointer(org.bytedeco.javacpp.IntPointer)

Example 2 with CompressedDataBuffer

use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.

the class NoOp method compressPointer.

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressionType(getCompressionType());
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setCompressedLength(length * elementSize);
    descriptor.setNumberOfElements(length);
    BytePointer ptr = new BytePointer(length * elementSize);
    // this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
    Pointer.memcpy(ptr, srcPointer, length * elementSize);
    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
    return buffer;
}
Also used : CompressionDescriptor(org.nd4j.linalg.compression.CompressionDescriptor) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer) BytePointer(org.bytedeco.javacpp.BytePointer)

Example 3 with CompressedDataBuffer

use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.

the class NoOp method compress.

@Override
public DataBuffer compress(DataBuffer buffer) {
    CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
    BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize());
    CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor);
    Nd4j.getMemoryManager().memcpy(result, buffer);
    return result;
}
Also used : CompressionDescriptor(org.nd4j.linalg.compression.CompressionDescriptor) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer) BytePointer(org.bytedeco.javacpp.BytePointer)

Example 4 with CompressedDataBuffer

use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.

the class Gzip method decompress.

@Override
public DataBuffer decompress(DataBuffer buffer) {
    try {
        CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
        CompressionDescriptor descriptor = compressed.getCompressionDescriptor();
        BytePointer pointer = (BytePointer) compressed.addressPointer();
        ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
        GZIPInputStream gzip = new GZIPInputStream(bis);
        DataInputStream dis = new DataInputStream(gzip);
        DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements());
        bufferRestored.read(dis);
        return bufferRestored;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Also used : GZIPInputStream(java.util.zip.GZIPInputStream) CompressionDescriptor(org.nd4j.linalg.compression.CompressionDescriptor) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer) ByteArrayInputStream(java.io.ByteArrayInputStream) BytePointer(org.bytedeco.javacpp.BytePointer) DataInputStream(java.io.DataInputStream) DataBuffer(org.nd4j.linalg.api.buffer.DataBuffer) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer)

Example 5 with CompressedDataBuffer

use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.

the class Int8 method compressPointer.

@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
    BytePointer ptr = new BytePointer(length);
    CompressionDescriptor descriptor = new CompressionDescriptor();
    descriptor.setCompressedLength(length * 1);
    descriptor.setOriginalLength(length * elementSize);
    descriptor.setOriginalElementSize(elementSize);
    descriptor.setNumberOfElements(length);
    descriptor.setCompressionAlgorithm(getDescriptor());
    descriptor.setCompressionType(getCompressionType());
    CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
    Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length);
    return buffer;
}
Also used : CompressionDescriptor(org.nd4j.linalg.compression.CompressionDescriptor) CompressedDataBuffer(org.nd4j.linalg.compression.CompressedDataBuffer) BytePointer(org.bytedeco.javacpp.BytePointer)

Aggregations

CompressedDataBuffer (org.nd4j.linalg.compression.CompressedDataBuffer)22 CompressionDescriptor (org.nd4j.linalg.compression.CompressionDescriptor)17 BytePointer (org.bytedeco.javacpp.BytePointer)10 DataBuffer (org.nd4j.linalg.api.buffer.DataBuffer)8 INDArray (org.nd4j.linalg.api.ndarray.INDArray)4 ByteBuffer (java.nio.ByteBuffer)3 IntPointer (org.bytedeco.javacpp.IntPointer)3 CudaDoubleDataBuffer (org.nd4j.linalg.jcublas.buffer.CudaDoubleDataBuffer)3 AllocationPoint (org.nd4j.jita.allocator.impl.AllocationPoint)2 CudaIntDataBuffer (org.nd4j.linalg.jcublas.buffer.CudaIntDataBuffer)2 CudaContext (org.nd4j.linalg.jcublas.context.CudaContext)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 DataInputStream (java.io.DataInputStream)1 DataOutputStream (java.io.DataOutputStream)1 ArrayList (java.util.ArrayList)1 GZIPInputStream (java.util.zip.GZIPInputStream)1 GZIPOutputStream (java.util.zip.GZIPOutputStream)1 ByteArrayOutputStream (org.apache.commons.io.output.ByteArrayOutputStream)1 Pointer (org.bytedeco.javacpp.Pointer)1 MatchCondition (org.nd4j.linalg.api.ops.impl.accum.MatchCondition)1