Examples with SimpleKey com.ms.silverking.cloud.dht.common.SimpleKey used on opensource projects

Example 1 with SimpleKey

use of com.ms.silverking.cloud.dht.common.SimpleKey in project SilverKing by Morgan-Stanley.

the class MD5KeyDigest method computeKey.

public DHTKey computeKey(String s) {
    MessageDigest md;
    md = MD5Digest.getLocalMessageDigest();
    for (int i = 0; i < s.length(); i++) {
        md.update((byte) s.charAt(i));
    }
    return new SimpleKey(md.digest());
}

Example 2 with SimpleKey

use of com.ms.silverking.cloud.dht.common.SimpleKey in project SilverKing by Morgan-Stanley.

the class KVSerializationTest method runTest.

public void runTest(int numKeys, BufferMode bufferMode) {
    SerializedKeyValueBuffers skvBuffers;
    Stopwatch sw;
    ThreadLocalRandom random;
    skvBuffers = new SerializedKeyValueBuffers();
    random = ThreadLocalRandom.current();
    sw = new SimpleStopwatch();
    for (int i = 0; i < numKeys; i++) {
        DHTKey key;
        ByteBuffer value;
        key = new SimpleKey(random.nextLong(), random.nextLong());
        value = randomValue(random);
        skvBuffers.addKeyValue(key, value, bufferMode);
    }
    sw.stop();
    System.out.println(skvBuffers);
    System.out.printf("Elapsed %f\n", sw.getElapsedSeconds());
    skvBuffers.freeze();
    display(skvBuffers);
}

Example 3 with SimpleKey

use of com.ms.silverking.cloud.dht.common.SimpleKey in project SilverKing by Morgan-Stanley.

the class IntArrayCuckoo method rehashAndAdd.

public static IntArrayCuckoo rehashAndAdd(IntArrayCuckoo oldTable, DHTKey key, int value) {
    IntArrayCuckoo newTable;
    newTable = rehash(oldTable);
    try {
        newTable.put(key, value);
        return newTable;
    } catch (TableFullException tfe) {
        // Something is very wrong if we get here.
        // For instance, are duplicate keys being put into the table?
        System.out.println("old table");
        oldTable.displaySizes();
        System.out.println("new table");
        newTable.displaySizes();
        System.out.println("\n\n\n");
        oldTable.display();
        System.out.println("\n\n");
        newTable.display();
        System.out.printf("\n%s\t%d\n", new SimpleKey(key).toString(), value);
        throw new RuntimeException("Unexpected table full after rehash: " + new SimpleKey(key).toString());
    }
}

Example 4 with SimpleKey

use of com.ms.silverking.cloud.dht.common.SimpleKey in project SilverKing by Morgan-Stanley.

the class AsyncPutOperationImpl method segment.

private void segment(K key, List<MessageGroup> messageGroups) {
    int numSegments;
    int valueSize;
    DHTKey dhtKey;
    DHTKey[] subKeys;
    ByteBuffer buf;
    ByteBuffer[] subBufs;
    ProtoPutMessageGroup<V> protoPutMG;
    Compression compression;
    SegmentedPutValue segmentedPutValue;
    boolean listenerInserted;
    int uncompressedLength;
    int storedLength;
    byte[] checksum;
    Log.fine("segmenting: ", key);
    // Serialize the value and compress if needed
    buf = nspoImpl.getValueSerializer().serializeToBuffer(putOperation.getValue(key));
    uncompressedLength = buf.limit();
    compression = putOperation.putOptions().getCompression();
    if (compression != Compression.NONE) {
        Compressor compressor;
        byte[] compressedValue;
        compressor = CodecProvider.getCompressor(compression);
        try {
            compressedValue = compressor.compress(buf.array(), buf.position(), buf.remaining());
            buf = ByteBuffer.wrap(compressedValue);
        } catch (IOException ioe) {
            throw new RuntimeException("Compression error in segmentation", ioe);
        }
    }
    storedLength = buf.limit();
    // Checksum the value
    // For segmented values we do not compute a complete checksum, but
    // instead we use the piecewise checksums. The primary reason for this is to allow for the
    // standard corrupt value detection/correction code to work for segmented values.
    checksum = new byte[putOperation.putOptions().getChecksumType().length()];
    // Now segment the value
    valueSize = buf.limit();
    numSegments = SegmentationUtil.getNumSegments(valueSize, SegmentationUtil.maxValueSegmentSize);
    segmentsCreated += numSegments;
    subBufs = new ByteBuffer[numSegments];
    for (int i = 0; i < numSegments; i++) {
        ByteBuffer subBuf;
        int segmentStart;
        int segmentSize;
        segmentStart = i * SegmentationUtil.maxValueSegmentSize;
        segmentSize = Math.min(SegmentationUtil.maxValueSegmentSize, valueSize - segmentStart);
        buf.position(segmentStart);
        subBuf = buf.slice();
        subBuf.limit(segmentSize);
        subBufs[i] = subBuf;
        if (debugSegmentation) {
            System.out.printf("%d\t%d\t%s\n", segmentStart, segmentSize, subBufs[i]);
        }
    }
    dhtKey = keyCreator.createKey(key);
    subKeys = keyCreator.createSubKeys(dhtKey, numSegments);
    if (segmentedPutValues == null) {
        segmentedPutValues = new LinkedList<>();
    }
    segmentedPutValue = new SegmentedPutValue(subKeys, dhtKey, this);
    segmentedPutValues.add(segmentedPutValue);
    // For now, assume only one segment per message
    for (int i = 0; i < numSegments; i++) {
        byte[] segmentChecksum;
        protoPutMG = createProtoPutMG(new PutMessageEstimate(1, subBufs[i].limit()));
        // hold a reference to the uuid to prevent GC
        opUUIDs.add((OperationUUID) protoPutMG.getUUID());
        listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), subKeys[i], segmentedPutValue);
        if (!listenerInserted) {
            throw new RuntimeException("Panic: Unable to insert listener into dedicated segment protoPutMG");
        }
        if (debugSegmentation) {
            System.out.printf("segmentation listener: %s\t%s\t%s\n", protoPutMG.getUUID(), subKeys[i], subBufs[i]);
        // System.out.printf("segmentation listener: %s\t%s\t%s\n",
        // protoPutMG.getUUID(), subKeys[i], StringUtil.byteBufferToHexString(subBufs[i]));
        }
        protoPutMG.addValueDedicated(subKeys[i], subBufs[i]);
        protoPutMG.addToMessageGroupList(messageGroups);
        segmentChecksum = new byte[putOperation.putOptions().getChecksumType().length()];
        protoPutMG.getMostRecentChecksum(segmentChecksum);
        ArrayUtil.xor(checksum, segmentChecksum);
    }
    // Now add the index key/value
    // indicate segmentation by storing segmentationBytes in the creator field
    protoPutMG = createProtoPutMG(new PutMessageEstimate(1, SegmentationUtil.segmentedValueBufferLength), MetaDataConstants.segmentationBytes);
    // hold a reference to the uuid to prevent GC
    opUUIDs.add((OperationUUID) protoPutMG.getUUID());
    listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), dhtKey, segmentedPutValue);
    if (!listenerInserted) {
        throw new RuntimeException("Panic: Unable to add index key/value into dedicated protoPutMG");
    }
    if (debug) {
        System.out.printf("added index listener %s %s\n", protoPutMG.getUUID(), new SimpleKey(dhtKey));
    }
    ByteBuffer segmentMetaDataBuffer;
    segmentMetaDataBuffer = SegmentationUtil.createSegmentMetaDataBuffer(DHTClient.getValueCreator().getBytes(), storedLength, uncompressedLength, putOperation.putOptions().getChecksumType(), checksum);
    protoPutMG.addValueDedicated(dhtKey, segmentMetaDataBuffer);
    protoPutMG.addToMessageGroupList(messageGroups);
}

Example 5 with SimpleKey

use of com.ms.silverking.cloud.dht.common.SimpleKey in project SilverKing by Morgan-Stanley.

the class ActivePutListeners method receivedPutResponse.

public <K, V> void receivedPutResponse(MessageGroup message) {
    if (enableMultipleOpsPerMessage) {
        long version;
        ConcurrentMap<DHTKey, WeakReference<ActiveKeyedOperationResultListener<OpResult>>> listenerMap;
        version = message.getBuffers()[0].getLong(0);
        // Log.warning("receivedPutResponse version ", version);
        listenerMap = activePutListeners.get(message.getUUID());
        if (listenerMap != null) {
            for (MessageGroupKeyOrdinalEntry entry : message.getKeyOrdinalIterator()) {
                WeakReference<ActiveKeyedOperationResultListener<OpResult>> listenerRef;
                ActiveKeyedOperationResultListener<OpResult> listener;
                if (debug) {
                    System.out.println(new SimpleKey(entry.getKey()));
                }
                listenerRef = listenerMap.get(entry.getKey());
                listener = listenerRef.get();
                if (listener != null) {
                    listener.resultReceived(entry.getKey(), EnumValues.opResult[entry.getOrdinal()]);
                } else {
                    Log.info("receivedPutResponse. null listener ref for: ", message.getUUID() + "\t" + entry.getKey());
                }
            }
        } else {
            // If we're receiving the error, then it's possible that we lost the
            // reference that was stored in the weak map.
            Log.warning("receivedPutResponse. No listenerMap for: ", message.getUUID());
        }
    } else {
        ActiveKeyedOperationResultListener<OpResult> listener;
        listener = activeOpListeners.get(message.getUUID());
        if (listener == null) {
            Log.info("receivedRetrievalResponse. No listener for uuid: ", message.getUUID());
        } else {
            for (MessageGroupKeyOrdinalEntry entry : message.getKeyOrdinalIterator()) {
                listener.resultReceived(entry.getKey(), EnumValues.opResult[entry.getOrdinal()]);
            }
        }
    }
}