use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.
the class CpuFlexibleThreshold method compress.
@Override
public DataBuffer compress(DataBuffer buffer) {
INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[] { 1, (int) buffer.length() }).getFirst());
double max = temp.amaxNumber().doubleValue();
int cntAbs = temp.scan(Conditions.absGreaterThanOrEqual(max - (max * threshold))).intValue();
long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
int compressedLength = cntAbs + 4;
// first 3 elements contain header
IntPointer pointer = new IntPointer(compressedLength);
pointer.put(0, cntAbs);
pointer.put(1, (int) buffer.length());
// please note, this value will be ovewritten anyway
pointer.put(2, Float.floatToIntBits(threshold));
pointer.put(3, 0);
CompressionDescriptor descriptor = new CompressionDescriptor();
// sizeOf(INT)
descriptor.setCompressedLength(compressedLength * 4);
descriptor.setOriginalLength(originalLength);
descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
descriptor.setNumberOfElements(buffer.length());
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setCompressionType(getCompressionType());
CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.FTHRESHOLD, pointer, buffer.length());
Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);
return cbuff;
}
use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.
the class CpuThreshold method compress.
@Override
public DataBuffer compress(DataBuffer buffer) {
INDArray temp = Nd4j.createArrayFromShapeBuffer(buffer, Nd4j.getShapeInfoProvider().createShapeInformation(new int[] { 1, (int) buffer.length() }).getFirst());
MatchCondition condition = new MatchCondition(temp, Conditions.absGreaterThanOrEqual(threshold));
int cntAbs = Nd4j.getExecutioner().exec(condition, Integer.MAX_VALUE).getInt(0);
if (cntAbs < 2)
return null;
long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
int compressedLength = cntAbs + 4;
// first 3 elements contain header
IntPointer pointer = new IntPointer(compressedLength);
pointer.put(0, cntAbs);
pointer.put(1, (int) buffer.length());
pointer.put(2, Float.floatToIntBits(threshold));
pointer.put(3, 0);
CompressionDescriptor descriptor = new CompressionDescriptor();
// sizeOf(INT)
descriptor.setCompressedLength(compressedLength * 4);
descriptor.setOriginalLength(originalLength);
descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
descriptor.setNumberOfElements(buffer.length());
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setCompressionType(getCompressionType());
CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.THRESHOLD, pointer, buffer.length());
Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);
return cbuff;
}
use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.
the class NativeOpExecutioner method thresholdEncode.
@Override
public INDArray thresholdEncode(INDArray input, double threshold, Integer boundary) {
MatchCondition condition = new MatchCondition(input, Conditions.absGreaterThanOrEqual(threshold));
int cntAbs = Nd4j.getExecutioner().exec(condition, Integer.MAX_VALUE).getInt(0);
if (cntAbs < 2)
return null;
if (boundary != null)
cntAbs = Math.min(cntAbs, boundary);
DataBuffer buffer = input.data();
long originalLength = buffer.length() * Nd4j.sizeOfDataType(buffer.dataType());
int compressedLength = cntAbs + 4;
// first 3 elements contain header
DataBuffer encodedBuffer = Nd4j.getMemoryManager().getCurrentWorkspace() == null ? Nd4j.getDataBufferFactory().createInt(4 + cntAbs, false) : Nd4j.getDataBufferFactory().createInt(4 + cntAbs, false, Nd4j.getMemoryManager().getCurrentWorkspace());
encodedBuffer.put(0, cntAbs);
encodedBuffer.put(1, (int) buffer.length());
encodedBuffer.put(2, Float.floatToIntBits((float) threshold));
// format id
encodedBuffer.put(3, ThresholdCompression.FLEXIBLE_ENCODING);
CompressionDescriptor descriptor = new CompressionDescriptor();
// sizeOf(INT)
descriptor.setCompressedLength(compressedLength * 4);
descriptor.setOriginalLength(originalLength);
descriptor.setOriginalElementSize(Nd4j.sizeOfDataType(buffer.dataType()));
descriptor.setNumberOfElements(buffer.length());
descriptor.setCompressionAlgorithm("THRESHOLD");
descriptor.setCompressionType(CompressionType.LOSSLESS);
// CompressedDataBuffer cbuff = new CompressedDataBuffer(pointer, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(AbstractCompressor.getBufferTypeEx(buffer), buffer.addressPointer(), DataBuffer.TypeEx.THRESHOLD, encodedBuffer.addressPointer(), buffer.length());
Nd4j.getAffinityManager().tagLocation(buffer, AffinityManager.Location.HOST);
return Nd4j.createArrayFromShapeBuffer(encodedBuffer, input.shapeInfoDataBuffer());
}
use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.
the class Float16 method compressPointer.
@Override
protected CompressedDataBuffer compressPointer(DataBuffer.TypeEx srcType, Pointer srcPointer, int length, int elementSize) {
BytePointer ptr = new BytePointer(length * 2);
CompressionDescriptor descriptor = new CompressionDescriptor();
descriptor.setCompressedLength(length * 2);
descriptor.setOriginalLength(length * elementSize);
descriptor.setOriginalElementSize(elementSize);
descriptor.setNumberOfElements(length);
descriptor.setCompressionAlgorithm(getDescriptor());
descriptor.setCompressionType(getCompressionType());
CompressedDataBuffer buffer = new CompressedDataBuffer(ptr, descriptor);
Nd4j.getNDArrayFactory().convertDataEx(srcType, srcPointer, DataBuffer.TypeEx.FLOAT16, ptr, length);
return buffer;
}
use of org.nd4j.linalg.compression.CompressionDescriptor in project nd4j by deeplearning4j.
the class Gzip method compress.
@Override
public DataBuffer compress(DataBuffer buffer) {
try {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(stream);
DataOutputStream dos = new DataOutputStream(gzip);
buffer.write(dos);
dos.flush();
dos.close();
byte[] bytes = stream.toByteArray();
// logger.info("Bytes: {}", Arrays.toString(bytes));
BytePointer pointer = new BytePointer(bytes);
CompressionDescriptor descriptor = new CompressionDescriptor(buffer, this);
descriptor.setCompressedLength(bytes.length);
CompressedDataBuffer result = new CompressedDataBuffer(pointer, descriptor);
return result;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
Aggregations