Examples with Compression com.ms.silverking.cloud.dht.client.Compression used on opensource projects

Example 1 with Compression

use of com.ms.silverking.cloud.dht.client.Compression in project SilverKing by Morgan-Stanley.

the class ValueUtil method verifyChecksum.

public static void verifyChecksum(ByteBuffer storedValue) throws CorruptValueException {
    if (storedValue != null) {
        int baseOffset;
        byte[] storedData;
        byte[] dataToVerify;
        int verifyDataOffset;
        int dataOffset;
        int compressedLength;
        int uncompressedLength;
        Compression compression;
        // don't alter the source buffer
        storedValue = storedValue.duplicate();
        baseOffset = storedValue.position();
        storedData = new byte[storedValue.limit()];
        storedValue.get(storedData);
        compressedLength = MetaDataUtil.getCompressedLength(storedData, baseOffset);
        uncompressedLength = MetaDataUtil.getUncompressedLength(storedData, baseOffset);
        if (debugChecksum) {
            System.out.println("compressedLength: " + compressedLength);
            System.out.println("uncompressedLength: " + uncompressedLength);
        }
        dataOffset = MetaDataUtil.getDataOffset(storedData, baseOffset);
        compression = EnumValues.compression[MetaDataUtil.getCompression(storedData, baseOffset)];
        if (MetaDataUtil.isCompressed(storedData, baseOffset)) {
            byte[] uncompressedData;
            Decompressor decompressor;
            Log.fine("Compressed");
            decompressor = CodecProvider.getDecompressor(compression);
            try {
                // System.out.println(compression +" "+ decompressor);
                uncompressedData = decompressor.decompress(storedData, dataOffset, compressedLength, uncompressedLength);
                dataToVerify = uncompressedData;
                verifyDataOffset = 0;
            } catch (Exception e) {
                throw new CorruptValueException(e);
            }
        } else {
            dataToVerify = storedData;
            verifyDataOffset = dataOffset;
        }
        verifyChecksum(storedData, baseOffset, dataToVerify, verifyDataOffset, uncompressedLength);
        Log.warningAsync("Checksum OK");
    }
}

Example 2 with Compression

use of com.ms.silverking.cloud.dht.client.Compression in project SilverKing by Morgan-Stanley.

the class RawRetrievalResult method setStoredValue.

public void setStoredValue(ByteBuffer storedValue, boolean verifyChecksum, boolean filterInvalidated, EncrypterDecrypter encrypterDecrypter) throws CorruptValueException {
    int baseOffset;
    byte[] storedData;
    byte[] dataToVerify;
    int verifyDataOffset;
    int dataOffset;
    int compressedLength;
    int uncompressedLength;
    Compression compression;
    setStoredValue_direct(storedValue);
    dataToVerify = null;
    verifyDataOffset = 0;
    if (result == OpResult.SUCCEEDED) {
        baseOffset = storedValue.position();
        storedData = storedValue.array();
        // System.out.printf("%d\t%d\n", baseOffset, storedData.length);
        compressedLength = MetaDataUtil.getCompressedLength(storedData, baseOffset);
        uncompressedLength = MetaDataUtil.getUncompressedLength(storedData, baseOffset);
        if (debugChecksum) {
            System.out.println("compressedLength: " + compressedLength);
            System.out.println("uncompressedLength: " + uncompressedLength);
        }
        dataOffset = MetaDataUtil.getDataOffset(storedData, baseOffset);
        if (retrievalType.hasValue()) {
            if (encrypterDecrypter != null) {
                byte[] _storedData;
                byte[] plainText;
                int offset;
                int tailLength;
                offset = MetaDataUtil.getDataOffset(storedData, baseOffset);
                plainText = encrypterDecrypter.decrypt(storedData, offset, compressedLength);
                dataToVerify = new byte[compressedLength];
                verifyDataOffset = 0;
                System.arraycopy(storedData, offset, dataToVerify, 0, compressedLength);
                tailLength = storedData.length - offset - compressedLength;
                _storedData = new byte[offset + plainText.length + tailLength];
                System.arraycopy(storedData, 0, _storedData, 0, offset);
                System.arraycopy(plainText, 0, _storedData, offset, plainText.length);
                System.arraycopy(storedData, offset + compressedLength, _storedData, offset + plainText.length, tailLength);
                storedData = _storedData;
            }
            compression = EnumValues.compression[MetaDataUtil.getCompression(storedData, baseOffset)];
            if (MetaDataUtil.isCompressed(storedData, baseOffset)) {
                byte[] uncompressedData;
                Decompressor decompressor;
                Log.fine("Compressed");
                decompressor = CodecProvider.getDecompressor(compression);
                if (decompressor == null) {
                    if (compression == Compression.NONE) {
                        throw new RuntimeException("MetaDataUtil.isCompressed() returning true for uncompressed data");
                    } else {
                        throw new RuntimeException("Can't find compressor for: " + compression);
                    }
                }
                try {
                    // System.out.println(compression +" "+ decompressor);
                    uncompressedData = decompressor.decompress(storedData, dataOffset, compressedLength, uncompressedLength);
                    if (encrypterDecrypter == null) {
                        dataToVerify = uncompressedData;
                        verifyDataOffset = 0;
                    }
                } catch (Exception e) {
                    if (Log.levelMet(Level.INFO)) {
                        Log.logErrorWarning(e);
                    }
                    throw new CorruptValueException(e);
                }
            } else {
                if (encrypterDecrypter == null) {
                    dataToVerify = storedData;
                    verifyDataOffset = dataOffset;
                }
            }
        } else {
            dataToVerify = storedData;
            verifyDataOffset = dataOffset;
        }
        if (filterInvalidated && ValueUtil.isInvalidated(storedData, baseOffset)) {
            result = OpResult.NO_SUCH_VALUE;
        } else {
            if (retrievalType.hasValue()) {
                if (verifyChecksum) {
                    int verifyDataLength;
                    verifyDataLength = dataToVerify.length - verifyDataOffset;
                    ValueUtil.verifyChecksum(storedData, baseOffset, dataToVerify, verifyDataOffset, verifyDataLength);
                }
                cookedValue = checkedWrap(dataToVerify, verifyDataOffset, uncompressedLength);
            // cookedValue = checkedWrap(storedData, dataOffset, uncompressedLength);
            }
        }
    }
}

Example 3 with Compression

use of com.ms.silverking.cloud.dht.client.Compression in project SilverKing by Morgan-Stanley.

the class AsyncPutOperationImpl method segment.

private void segment(K key, List<MessageGroup> messageGroups) {
    int numSegments;
    int valueSize;
    DHTKey dhtKey;
    DHTKey[] subKeys;
    ByteBuffer buf;
    ByteBuffer[] subBufs;
    ProtoPutMessageGroup<V> protoPutMG;
    Compression compression;
    SegmentedPutValue segmentedPutValue;
    boolean listenerInserted;
    int uncompressedLength;
    int storedLength;
    byte[] checksum;
    Log.fine("segmenting: ", key);
    // Serialize the value and compress if needed
    buf = nspoImpl.getValueSerializer().serializeToBuffer(putOperation.getValue(key));
    uncompressedLength = buf.limit();
    compression = putOperation.putOptions().getCompression();
    if (compression != Compression.NONE) {
        Compressor compressor;
        byte[] compressedValue;
        compressor = CodecProvider.getCompressor(compression);
        try {
            compressedValue = compressor.compress(buf.array(), buf.position(), buf.remaining());
            buf = ByteBuffer.wrap(compressedValue);
        } catch (IOException ioe) {
            throw new RuntimeException("Compression error in segmentation", ioe);
        }
    }
    storedLength = buf.limit();
    // Checksum the value
    // For segmented values we do not compute a complete checksum, but
    // instead we use the piecewise checksums. The primary reason for this is to allow for the
    // standard corrupt value detection/correction code to work for segmented values.
    checksum = new byte[putOperation.putOptions().getChecksumType().length()];
    // Now segment the value
    valueSize = buf.limit();
    numSegments = SegmentationUtil.getNumSegments(valueSize, SegmentationUtil.maxValueSegmentSize);
    segmentsCreated += numSegments;
    subBufs = new ByteBuffer[numSegments];
    for (int i = 0; i < numSegments; i++) {
        ByteBuffer subBuf;
        int segmentStart;
        int segmentSize;
        segmentStart = i * SegmentationUtil.maxValueSegmentSize;
        segmentSize = Math.min(SegmentationUtil.maxValueSegmentSize, valueSize - segmentStart);
        buf.position(segmentStart);
        subBuf = buf.slice();
        subBuf.limit(segmentSize);
        subBufs[i] = subBuf;
        if (debugSegmentation) {
            System.out.printf("%d\t%d\t%s\n", segmentStart, segmentSize, subBufs[i]);
        }
    }
    dhtKey = keyCreator.createKey(key);
    subKeys = keyCreator.createSubKeys(dhtKey, numSegments);
    if (segmentedPutValues == null) {
        segmentedPutValues = new LinkedList<>();
    }
    segmentedPutValue = new SegmentedPutValue(subKeys, dhtKey, this);
    segmentedPutValues.add(segmentedPutValue);
    // For now, assume only one segment per message
    for (int i = 0; i < numSegments; i++) {
        byte[] segmentChecksum;
        protoPutMG = createProtoPutMG(new PutMessageEstimate(1, subBufs[i].limit()));
        // hold a reference to the uuid to prevent GC
        opUUIDs.add((OperationUUID) protoPutMG.getUUID());
        listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), subKeys[i], segmentedPutValue);
        if (!listenerInserted) {
            throw new RuntimeException("Panic: Unable to insert listener into dedicated segment protoPutMG");
        }
        if (debugSegmentation) {
            System.out.printf("segmentation listener: %s\t%s\t%s\n", protoPutMG.getUUID(), subKeys[i], subBufs[i]);
        // System.out.printf("segmentation listener: %s\t%s\t%s\n",
        // protoPutMG.getUUID(), subKeys[i], StringUtil.byteBufferToHexString(subBufs[i]));
        }
        protoPutMG.addValueDedicated(subKeys[i], subBufs[i]);
        protoPutMG.addToMessageGroupList(messageGroups);
        segmentChecksum = new byte[putOperation.putOptions().getChecksumType().length()];
        protoPutMG.getMostRecentChecksum(segmentChecksum);
        ArrayUtil.xor(checksum, segmentChecksum);
    }
    // Now add the index key/value
    // indicate segmentation by storing segmentationBytes in the creator field
    protoPutMG = createProtoPutMG(new PutMessageEstimate(1, SegmentationUtil.segmentedValueBufferLength), MetaDataConstants.segmentationBytes);
    // hold a reference to the uuid to prevent GC
    opUUIDs.add((OperationUUID) protoPutMG.getUUID());
    listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), dhtKey, segmentedPutValue);
    if (!listenerInserted) {
        throw new RuntimeException("Panic: Unable to add index key/value into dedicated protoPutMG");
    }
    if (debug) {
        System.out.printf("added index listener %s %s\n", protoPutMG.getUUID(), new SimpleKey(dhtKey));
    }
    ByteBuffer segmentMetaDataBuffer;
    segmentMetaDataBuffer = SegmentationUtil.createSegmentMetaDataBuffer(DHTClient.getValueCreator().getBytes(), storedLength, uncompressedLength, putOperation.putOptions().getChecksumType(), checksum);
    protoPutMG.addValueDedicated(dhtKey, segmentMetaDataBuffer);
    protoPutMG.addToMessageGroupList(messageGroups);
}

Example 4 with Compression

use of com.ms.silverking.cloud.dht.client.Compression in project SilverKing by Morgan-Stanley.

the class DevTest method main.

/**
 * @param args
 */
public static void main(String[] args) {
    try {
        if (args.length != 9) {
            System.out.println("args: <zkLocs> <dhtName> <valueSize> <numKeys> <reps> <namespace> <compression> <checksum> <tests>");
            return;
        } else {
            HostAndPort[] zkLocs;
            String dhtName;
            int valueSize;
            int numKeys;
            int reps;
            String namespace;
            Compression compression;
            ChecksumType checksumType;
            EnumSet<Test> tests;
            LWTPoolProvider.createDefaultWorkPools();
            zkLocs = HostAndPort.parseMultiple(args[0]);
            dhtName = args[1];
            valueSize = Integer.parseInt(args[2]);
            numKeys = Integer.parseInt(args[3]);
            reps = Integer.parseInt(args[4]);
            namespace = args[5];
            compression = Compression.valueOf(args[6]);
            checksumType = ChecksumType.valueOf(args[7]);
            tests = parseTests(args[8]);
            // Log.setLevelAll();
            new DevTest(zkLocs, dhtName, valueSize).test(numKeys, reps, namespace, compression, checksumType, tests);
        // ThreadUtil.sleep(60000);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}