use of com.ms.silverking.compression.Compressor in project SilverKing by Morgan-Stanley.
the class AsyncPutOperationImpl method segment.
private void segment(K key, List<MessageGroup> messageGroups) {
int numSegments;
int valueSize;
DHTKey dhtKey;
DHTKey[] subKeys;
ByteBuffer buf;
ByteBuffer[] subBufs;
ProtoPutMessageGroup<V> protoPutMG;
Compression compression;
SegmentedPutValue segmentedPutValue;
boolean listenerInserted;
int uncompressedLength;
int storedLength;
byte[] checksum;
Log.fine("segmenting: ", key);
// Serialize the value and compress if needed
buf = nspoImpl.getValueSerializer().serializeToBuffer(putOperation.getValue(key));
uncompressedLength = buf.limit();
compression = putOperation.putOptions().getCompression();
if (compression != Compression.NONE) {
Compressor compressor;
byte[] compressedValue;
compressor = CodecProvider.getCompressor(compression);
try {
compressedValue = compressor.compress(buf.array(), buf.position(), buf.remaining());
buf = ByteBuffer.wrap(compressedValue);
} catch (IOException ioe) {
throw new RuntimeException("Compression error in segmentation", ioe);
}
}
storedLength = buf.limit();
// Checksum the value
// For segmented values we do not compute a complete checksum, but
// instead we use the piecewise checksums. The primary reason for this is to allow for the
// standard corrupt value detection/correction code to work for segmented values.
checksum = new byte[putOperation.putOptions().getChecksumType().length()];
// Now segment the value
valueSize = buf.limit();
numSegments = SegmentationUtil.getNumSegments(valueSize, SegmentationUtil.maxValueSegmentSize);
segmentsCreated += numSegments;
subBufs = new ByteBuffer[numSegments];
for (int i = 0; i < numSegments; i++) {
ByteBuffer subBuf;
int segmentStart;
int segmentSize;
segmentStart = i * SegmentationUtil.maxValueSegmentSize;
segmentSize = Math.min(SegmentationUtil.maxValueSegmentSize, valueSize - segmentStart);
buf.position(segmentStart);
subBuf = buf.slice();
subBuf.limit(segmentSize);
subBufs[i] = subBuf;
if (debugSegmentation) {
System.out.printf("%d\t%d\t%s\n", segmentStart, segmentSize, subBufs[i]);
}
}
dhtKey = keyCreator.createKey(key);
subKeys = keyCreator.createSubKeys(dhtKey, numSegments);
if (segmentedPutValues == null) {
segmentedPutValues = new LinkedList<>();
}
segmentedPutValue = new SegmentedPutValue(subKeys, dhtKey, this);
segmentedPutValues.add(segmentedPutValue);
// For now, assume only one segment per message
for (int i = 0; i < numSegments; i++) {
byte[] segmentChecksum;
protoPutMG = createProtoPutMG(new PutMessageEstimate(1, subBufs[i].limit()));
// hold a reference to the uuid to prevent GC
opUUIDs.add((OperationUUID) protoPutMG.getUUID());
listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), subKeys[i], segmentedPutValue);
if (!listenerInserted) {
throw new RuntimeException("Panic: Unable to insert listener into dedicated segment protoPutMG");
}
if (debugSegmentation) {
System.out.printf("segmentation listener: %s\t%s\t%s\n", protoPutMG.getUUID(), subKeys[i], subBufs[i]);
// System.out.printf("segmentation listener: %s\t%s\t%s\n",
// protoPutMG.getUUID(), subKeys[i], StringUtil.byteBufferToHexString(subBufs[i]));
}
protoPutMG.addValueDedicated(subKeys[i], subBufs[i]);
protoPutMG.addToMessageGroupList(messageGroups);
segmentChecksum = new byte[putOperation.putOptions().getChecksumType().length()];
protoPutMG.getMostRecentChecksum(segmentChecksum);
ArrayUtil.xor(checksum, segmentChecksum);
}
// Now add the index key/value
// indicate segmentation by storing segmentationBytes in the creator field
protoPutMG = createProtoPutMG(new PutMessageEstimate(1, SegmentationUtil.segmentedValueBufferLength), MetaDataConstants.segmentationBytes);
// hold a reference to the uuid to prevent GC
opUUIDs.add((OperationUUID) protoPutMG.getUUID());
listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), dhtKey, segmentedPutValue);
if (!listenerInserted) {
throw new RuntimeException("Panic: Unable to add index key/value into dedicated protoPutMG");
}
if (debug) {
System.out.printf("added index listener %s %s\n", protoPutMG.getUUID(), new SimpleKey(dhtKey));
}
ByteBuffer segmentMetaDataBuffer;
segmentMetaDataBuffer = SegmentationUtil.createSegmentMetaDataBuffer(DHTClient.getValueCreator().getBytes(), storedLength, uncompressedLength, putOperation.putOptions().getChecksumType(), checksum);
protoPutMG.addValueDedicated(dhtKey, segmentMetaDataBuffer);
protoPutMG.addToMessageGroupList(messageGroups);
}
Aggregations