Examples with DHTKey com.ms.silverking.cloud.dht.common.DHTKey used on opensource projects

Example 21 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class MessageGroupKeyEntry method equals.

@Override
public boolean equals(Object o) {
    DHTKey oKey;
    oKey = (DHTKey) o;
    return lsl == oKey.getLSL() && msl == oKey.getMSL();
}

Example 22 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class ActiveRetrievalListeners method receivedRetrievalResponse.

void receivedRetrievalResponse(MessageGroup message) {
    if (enableMultipleOpsPerMessage) {
        // long    version;
        ConcurrentMap<DHTKey, List<WeakReference<ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry>>>> listenerMap;
        // version = message.getBuffers()[0].getLong(0);
        // System.out.println("receivedRetrievalResponse version "+ version);
        listenerMap = activeRetrievalListeners.get(message.getUUID());
        if (listenerMap != null) {
            // System.out.println("listenerMap "+ listenerMap);
            for (MessageGroupRetrievalResponseEntry entry : message.getRetrievalResponseValueKeyIterator()) {
                List<WeakReference<ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry>>> listenerList;
                // System.out.println("entry:\t"+ entry);
                listenerList = listenerMap.get(entry);
                if (listenerList != null) {
                    for (WeakReference<ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry>> listenerRef : listenerList) {
                        ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry> listener;
                        listener = listenerRef.get();
                        if (listener != null) {
                            listener.resultReceived(entry, entry);
                        } else {
                            Log.info("receivedRetrievalResponse. null listenerRef.get() for entry: ", entry);
                        }
                    }
                } else {
                    Log.warning("receivedRetrievalResponse. No listener for entry: ", entry);
                }
            }
        } else {
            // If we're receiving the error, then it's possible that we lost the
            // reference that was stored in the weak map.
            // FUTURE - was WARNING, think about level
            Log.info("receivedRetrievalResponse. No listenerMap for: ", message.getUUID());
        }
    } else {
        ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry> listener;
        listener = activeOpListeners.get(message.getUUID());
        if (listener == null) {
            Log.info("receivedRetrievalResponse. No listener for uuid: ", message.getUUID());
        } else {
            for (MessageGroupRetrievalResponseEntry entry : message.getRetrievalResponseValueKeyIterator()) {
                listener.resultReceived(entry, entry);
            }
        }
    }
}

Example 23 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class AsyncPutOperationImpl method segment.

private void segment(K key, List<MessageGroup> messageGroups) {
    int numSegments;
    int valueSize;
    DHTKey dhtKey;
    DHTKey[] subKeys;
    ByteBuffer buf;
    ByteBuffer[] subBufs;
    ProtoPutMessageGroup<V> protoPutMG;
    Compression compression;
    SegmentedPutValue segmentedPutValue;
    boolean listenerInserted;
    int uncompressedLength;
    int storedLength;
    byte[] checksum;
    Log.fine("segmenting: ", key);
    // Serialize the value and compress if needed
    buf = nspoImpl.getValueSerializer().serializeToBuffer(putOperation.getValue(key));
    uncompressedLength = buf.limit();
    compression = putOperation.putOptions().getCompression();
    if (compression != Compression.NONE) {
        Compressor compressor;
        byte[] compressedValue;
        compressor = CodecProvider.getCompressor(compression);
        try {
            compressedValue = compressor.compress(buf.array(), buf.position(), buf.remaining());
            buf = ByteBuffer.wrap(compressedValue);
        } catch (IOException ioe) {
            throw new RuntimeException("Compression error in segmentation", ioe);
        }
    }
    storedLength = buf.limit();
    // Checksum the value
    // For segmented values we do not compute a complete checksum, but
    // instead we use the piecewise checksums. The primary reason for this is to allow for the
    // standard corrupt value detection/correction code to work for segmented values.
    checksum = new byte[putOperation.putOptions().getChecksumType().length()];
    // Now segment the value
    valueSize = buf.limit();
    numSegments = SegmentationUtil.getNumSegments(valueSize, SegmentationUtil.maxValueSegmentSize);
    segmentsCreated += numSegments;
    subBufs = new ByteBuffer[numSegments];
    for (int i = 0; i < numSegments; i++) {
        ByteBuffer subBuf;
        int segmentStart;
        int segmentSize;
        segmentStart = i * SegmentationUtil.maxValueSegmentSize;
        segmentSize = Math.min(SegmentationUtil.maxValueSegmentSize, valueSize - segmentStart);
        buf.position(segmentStart);
        subBuf = buf.slice();
        subBuf.limit(segmentSize);
        subBufs[i] = subBuf;
        if (debugSegmentation) {
            System.out.printf("%d\t%d\t%s\n", segmentStart, segmentSize, subBufs[i]);
        }
    }
    dhtKey = keyCreator.createKey(key);
    subKeys = keyCreator.createSubKeys(dhtKey, numSegments);
    if (segmentedPutValues == null) {
        segmentedPutValues = new LinkedList<>();
    }
    segmentedPutValue = new SegmentedPutValue(subKeys, dhtKey, this);
    segmentedPutValues.add(segmentedPutValue);
    // For now, assume only one segment per message
    for (int i = 0; i < numSegments; i++) {
        byte[] segmentChecksum;
        protoPutMG = createProtoPutMG(new PutMessageEstimate(1, subBufs[i].limit()));
        // hold a reference to the uuid to prevent GC
        opUUIDs.add((OperationUUID) protoPutMG.getUUID());
        listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), subKeys[i], segmentedPutValue);
        if (!listenerInserted) {
            throw new RuntimeException("Panic: Unable to insert listener into dedicated segment protoPutMG");
        }
        if (debugSegmentation) {
            System.out.printf("segmentation listener: %s\t%s\t%s\n", protoPutMG.getUUID(), subKeys[i], subBufs[i]);
        // System.out.printf("segmentation listener: %s\t%s\t%s\n",
        // protoPutMG.getUUID(), subKeys[i], StringUtil.byteBufferToHexString(subBufs[i]));
        }
        protoPutMG.addValueDedicated(subKeys[i], subBufs[i]);
        protoPutMG.addToMessageGroupList(messageGroups);
        segmentChecksum = new byte[putOperation.putOptions().getChecksumType().length()];
        protoPutMG.getMostRecentChecksum(segmentChecksum);
        ArrayUtil.xor(checksum, segmentChecksum);
    }
    // Now add the index key/value
    // indicate segmentation by storing segmentationBytes in the creator field
    protoPutMG = createProtoPutMG(new PutMessageEstimate(1, SegmentationUtil.segmentedValueBufferLength), MetaDataConstants.segmentationBytes);
    // hold a reference to the uuid to prevent GC
    opUUIDs.add((OperationUUID) protoPutMG.getUUID());
    listenerInserted = activePutListeners.addListener(protoPutMG.getUUID(), dhtKey, segmentedPutValue);
    if (!listenerInserted) {
        throw new RuntimeException("Panic: Unable to add index key/value into dedicated protoPutMG");
    }
    if (debug) {
        System.out.printf("added index listener %s %s\n", protoPutMG.getUUID(), new SimpleKey(dhtKey));
    }
    ByteBuffer segmentMetaDataBuffer;
    segmentMetaDataBuffer = SegmentationUtil.createSegmentMetaDataBuffer(DHTClient.getValueCreator().getBytes(), storedLength, uncompressedLength, putOperation.putOptions().getChecksumType(), checksum);
    protoPutMG.addValueDedicated(dhtKey, segmentMetaDataBuffer);
    protoPutMG.addToMessageGroupList(messageGroups);
}

Example 24 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class AsyncRetrievalOperationImpl method resultReceived.

@Override
public void resultReceived(DHTKey dhtKey, MessageGroupRetrievalResponseEntry entry) {
    RawRetrievalResult rawResult;
    OpResult opResult;
    boolean setComplete;
    int oldSegmentsCreated;
    boolean segmented;
    // NEED TO MODIFY THIS METHOD TO ACCEPT SEGMENTED COMPLETIONS
    // THINK ABOUT STRUCTURE OF CODE
    oldSegmentsCreated = segmentsCreated;
    // System.out.printf("resultReceived key: %s\nentry: %s\n%s\nvalue: %s\n%s\n",
    // dhtKey, entry, entry.getOpResult(), StringUtil.byteBufferToHexString(entry.getValue()),
    // StringUtil.byteBufferToString(entry.getValue()));
    // System.out.printf("resultReceived key: %s\nentry: %s\n%s\nvalue buf: %s\n",
    // dhtKey, entry, entry.getOpResult(), entry.getValue());
    /*
        if (mvLock != null) {
            mvLock.lock();
        }
        try {
        */
    segmented = false;
    // FUTURE - avoid the multi-step construction
    rawResult = new RawRetrievalResult(retrievalOperation.retrievalOptions().getRetrievalType());
    opResult = entry.getOpResult();
    Log.fine("opResult: ", opResult);
    if (opResult == OpResult.SUCCEEDED) {
        try {
            if (testReceiveCorruption) {
                MetaDataUtil.testCorruption(entry.getValue(), receiveCorruptionProbability, MetaDataUtil.getDataOffset(entry.getValue(), 0));
            }
            // segmented = MetaDataUtil.isSegmented(entry.getValue().array(), entry.getValue().position());
            segmented = MetaDataUtil.isSegmented(entry.getValue());
            rawResult.setStoredValue(entry.getValue(), !segmented && retrievalOperation.retrievalOptions().getVerifyChecksums(), !retrievalOperation.retrievalOptions().getReturnInvalidations(), nspoImpl.getNSPOptions().getEncrypterDecrypter());
        } catch (CorruptValueException cve) {
            Log.infoAsync(String.format("Corrupt\t%s", dhtKey));
            handleCorruptValue(dhtKey);
            return;
        }
    } else {
        rawResult.setOpResult(opResult);
    }
    if (opResult == OpResult.SUCCEEDED && segmented) {
        ByteBuffer buf;
        if (debugSegmentation) {
            System.out.printf("SEGMENTED RESULT\n");
        }
        buf = rawResult.getValue();
        if (retrievalOperation.retrievalOptions().getRetrievalType().hasValue()) {
            DHTKey[] segmentKeys;
            int numSegments;
            if (debugSegmentation) {
                System.out.printf("SEGMENTED\t%s\t%s\t%d\t%d\n", StringUtil.byteArrayToHexString(SegmentationUtil.getCreatorBytes(buf)), buf, SegmentationUtil.getStoredLength(buf), SegmentationUtil.getUncompressedLength(buf));
            }
            if (true) {
                int storedLength;
                // not using below since the internal checksum should handle this
                // if (SegmentationUtil.checksumSegmentMetaDataBuffer(buf, nspoImpl.getNSPOptions().getChecksumType())) {
                setComplete = false;
                storedLength = SegmentationUtil.getStoredLength(buf);
                if (storedLength < 0) {
                    System.out.println(StringUtil.byteBufferToHexString(buf));
                    System.out.println(SegmentationUtil.getMetaData(rawResult, buf));
                    System.exit(-1);
                    numSegments = 1;
                } else {
                    numSegments = SegmentationUtil.getNumSegments(storedLength, SegmentationUtil.maxValueSegmentSize);
                }
                segmentsCreated += numSegments;
                if (debugSegmentation) {
                    System.out.printf("NUM SEGMENTS\t%d\n", numSegments);
                }
                segmentKeys = keyCreator.createSubKeys(dhtKey, numSegments);
                retrieveSegments(dhtKey, segmentKeys, SegmentationUtil.getMetaData(rawResult, buf));
            } else {
                setComplete = true;
                rawResult.setOpResult(OpResult.CORRUPT, true);
                opResult = OpResult.CORRUPT;
            }
        } else {
            if (debugSegmentation) {
                System.out.printf("SEGMENTED. MetaData retrieval\n");
            }
            setComplete = true;
        }
    } else {
        if (debugSegmentation) {
            System.out.printf("opResult %s\n", opResult);
        }
        setComplete = true;
    }
    if (setComplete) {
        RetrievalResultBase<V> prev;
        RetrievalResult<V> newResult;
        if (Log.levelMet(Level.FINE)) {
            Log.fine("setComplete: ", setComplete);
            Log.fine("dhtKey ", dhtKey);
        }
        newResult = new RetrievalResult<>(rawResult, nspoImpl.getValueDeserializer());
        prev = results.putIfAbsent(dhtKeyToKey.get(dhtKey), newResult);
        if (prev == null) {
            if (resultsReceived.incrementAndGet() >= size) {
                checkForCompletion();
            // FUTURE - this doesn't work for multi valued since we don't know how many we are getting...
            // For now, we ignore this since we aren't supporting multi-value yet
            }
        } else {
        /*
                RetrievalResultBase<V>  p;
                boolean					unique;
            	
                // FUTURE - handle mutual exclusion
                unique = true;
                p = prev;
                while (p.getNext() != null) {
                	p = p.getNext();
                	if (p.getCreationTime().equals(newResult.getCreationTime())) {
                		unique = false;
                		break;
                	}
                }
            	if (unique) {
            		p.setNext(newResult);
            	} else {
            		// Ignoring duplicate result
            	}
            	*/
        }
    }
    /*
        if (segmentsCreated != oldSegmentsCreated) {
            recomputeTimeoutState();
        }
        // FUTURE THINK ABOUT WHETHER WE NEED THIS
        */
    checkForUpdates();
}

Example 25 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class AsyncRetrievalOperationImpl method createMessagesForIncomplete.

private ProtoMessageGroup createMessagesForIncomplete(ProtoRetrievalMessageGroup protoMG, List<MessageGroup> messageGroups, KeyedMessageEstimate estimate) {
    int pmgRetrievals;
    // will be needed when we check for compatible retrievals
    int keysRemaining;
    pmgRetrievals = 0;
    keysRemaining = retrievalOperation.size();
    // now fill in keys and values
    for (K key : getKeys()) {
        if (!getSent() || OpResult.isIncompleteOrNull(getOpResult(key))) {
            DHTKey dhtKey;
            dhtKey = keyToDHTKey.get(key);
            // System.out.printf("%d\t%d %d\n", keysRemaining, pmgRetrievals, maxRetrievalsPerMessageGroup);
            if (activeRetrievalListeners.addListener(protoMG.getUUID(), dhtKey, this)) {
                protoMG.addKey(dhtKey);
            }
            // hold a reference to the uuid to prevent GC
            opUUIDs.add((OperationUUID) protoMG.getUUID());
            ++pmgRetrievals;
            --keysRemaining;
        } else {
            throw new RuntimeException("resends not yet implemented");
        }
    }
    return protoMG;
}