use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.
the class CudaFlexibleThreshold method compress.
@Override
public DataBuffer compress(DataBuffer buffer) {
INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[] { 1, (int) buffer.length() }));
double max = temp.amaxNumber().doubleValue();
int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();
long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
int compressedLength = cntAbs + 3;
// first 3 elements contain header
IntPointer pointer = new IntPointer(compressedLength);
pointer.put(0, cntAbs);
pointer.put(1, (int) buffer.length());
// please note, this value will be ovewritten anyway
pointer.put(2, Float.floatToIntBits(threshold));
CompressionDescriptor descriptor = new CompressionDescriptor();
// sizeOf(INT)
descriptor.setCompressedLength(compressedLength * 4);
descriptor.setOriginalLength(originalLength);
descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
descriptor.setNumberOfElements(buffer.length());
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setCompressionType(getCompressionType());
CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());
Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);
return cbuff;
}
use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.
the class NoOp method compressPointer.
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
CompressionDescriptor descriptor = new CompressionDescriptor();
descriptor.setCompressionType(getCompressionType());
descriptor.setOriginalLength(length * elementSize);
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setOriginalElementSize(elementSize);
descriptor.setCompressedLength(length * elementSize);
descriptor.setNumberOfElements(length);
BytePointer ptr = new BytePointer(length * elementSize);
// this Pointer.memcpy is used intentionally. This method operates on host memory ALWAYS
Pointer.memcpy(ptr, srcPointer, length * elementSize);
CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
return buffer;
}
use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.
the class NoOp method compress.
@Override
public DataBuffer compress(DataBuffer buffer) {
CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
BytePointer ptr = new BytePointer(buffer.length() * buffer.getElementSize());
CompressedDataBuffer result = new CompressedDataBuffer(ptr, descriptor);
Nd4j.getMemoryManager().memcpy(result, buffer);
return result;
}
use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.
the class Gzip method decompress.
@Override
public DataBuffer decompress(DataBuffer buffer) {
try {
CompressedDataBuffer compressed = (CompressedDataBuffer) buffer;
CompressionDescriptor descriptor = compressed.getCompressionDescriptor();
BytePointer pointer = (BytePointer) compressed.addressPointer();
ByteArrayInputStream bis = new ByteArrayInputStream(pointer.getStringBytes());
GZIPInputStream gzip = new GZIPInputStream(bis);
DataInputStream dis = new DataInputStream(gzip);
DataBuffer bufferRestored = Nd4j.createBuffer(descriptor.getNumberOfElements());
bufferRestored.read(dis);
return bufferRestored;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
use of org.nd4j.linalg.compression.CompressedDataBuffer in project nd4j by deeplearning4j.
the class Int8 method compressPointer.
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
BytePointer ptr = new BytePointer(length);
CompressionDescriptor descriptor = new CompressionDescriptor();
descriptor.setCompressedLength(length * 1);
descriptor.setOriginalLength(length * elementSize);
descriptor.setOriginalElementSize(elementSize);
descriptor.setNumberOfElements(length);
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setCompressionType(getCompressionType());
CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.INT8, ptr, length);
return buffer;
}
Aggregations