Examples with DHTKey com.ms.silverking.cloud.dht.common.DHTKey used on opensource projects

Example 31 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class LeafChecksumNode method computeChecksum.

private static ConvergenceChecksum computeChecksum(List<KeyAndVersionChecksum> keyAndVersionChecksums) {
    ConvergenceChecksum checksum;
    // checksum = new byte[keyValueChecksums.get(0).getValueChecksum().length];
    checksum = null;
    for (KeyAndVersionChecksum keyAndVersionChecksum : keyAndVersionChecksums) {
        DHTKey key;
        key = keyAndVersionChecksum.getKey();
        if (checksum == null) {
            checksum = new ConvergenceChecksum(keyAndVersionChecksum);
        } else {
            checksum = checksum.xor(new ConvergenceChecksum(keyAndVersionChecksum));
        }
    }
    return checksum;
}

Example 32 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method incomingChecksumTree.

// FUTURE - ENSURE THAT NO CONVERGENCE CODE USES THE RECEIVE THREADS TO DO WORK
// USE A WORKER
private void incomingChecksumTree(UUIDBase incomingUUID, ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    outstandingChecksumTreeRequests.remove(incomingUUID);
    checkQueuedChecksumTreeRequests();
    if (remoteTree == null) {
        checkForCompletion();
        return;
    }
    localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s", incomingUUID);
    }
    if (debug) {
        Log.warningAsyncf("incomingChecksumTree %x\t%s\t%s\t%s", ns, cp, remoteTree.getRegion(), incomingUUID);
        Log.warningAsyncf(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (debug) {
        Log.warningAsyncf("%s", matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
        Queue<DHTKey>   keysToFetchQueue;
        
        keysToFetchQueue = keysToFetchMap.get(cp);
        if (keysToFetchQueue == null) {
            Queue<DHTKey>   prev;
            
            keysToFetchQueue = new ConcurrentLinkedQueue<>();
            prev = keysToFetchMap.putIfAbsent(cp, keysToFetchQueue);
            if (prev != null) {
                keysToFetchQueue = prev;
            }
        }
        keysToFetchQueue.addAll(keysToFetch);
        */
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup        mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        convergenceControllers.put(uuid, this);
        sendSyncRetrievalRequest(srr);
    // mg = new ProtoRetrievalMessageGroup(uuid, ns, new InternalRetrievalOptions(retrievalOptions),
    // mgBase.getMyID(), batchKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(connection.getRemoteSocketAddress())));
    /*
            try {
                connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
            } catch (IOException ioe) {
                Log.logErrorWarning(ioe);
            }
            */
    // if (keysToFetchList.size() > 0) {
    // ThreadUtil.sleep(5);
    // }
    }
    checkMGQueue();
    if (debug) {
        Log.warningAsyncf("no more keysToFetch");
    }
    checkForCompletion();
}

Example 33 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class ActiveProxyRetrieval method sendResults.

/**
 * Send responses for all locally completed operations
 * @param retrievalOperation
 */
protected void sendResults(List<RetrievalResult> results) {
    if (debug) {
        System.out.printf("ActiveProxyRetrieval.sendresults() %d\n", results.size());
    }
    /*
        debugString.append("***\n");
        for (RetrievalResult result : results) {
            debugString.append(result.getValue() +"\n");
        }
        debugString.append("+++\n");
        */
    try {
        if (results.size() > 0) {
            ProtoValueMessageGroup pmg;
            byte[] _originator;
            List<List<RetrievalResult>> resultGroups;
            _originator = ConvergenceController2.isChecksumVersionConstraint(retrievalOptions.getVersionConstraint()) ? originator : messageModule.getMessageGroupBase().getMyID();
            resultGroups = createResultGroups(results);
            for (List<RetrievalResult> resultGroup : resultGroups) {
                MessageGroup messageGroup;
                int groupLength;
                groupLength = RetrievalResult.totalResultLength(resultGroup);
                pmg = new ProtoValueMessageGroup(uuid, namespace, results.size(), groupLength, _originator, messageModule.getAbsMillisTimeSource().relMillisRemaining(absDeadlineMillis));
                for (RetrievalResult result : resultGroup) {
                    DHTKey key;
                    ByteBuffer value;
                    if (debug) {
                        System.out.println(result);
                    }
                    key = result.getKey();
                    if (retrievalOptions.getWaitMode() != WaitMode.WAIT_FOR || result.getValue() == null) {
                        value = result.getValue();
                    } else {
                        value = result.getValue().duplicate();
                    }
                    if (value == null) {
                        pmg.addErrorCode(key, result.getResult());
                    } else {
                        pmg.addValue(key, value, result.getResultLength(), true);
                    }
                }
                messageGroup = pmg.toMessageGroup();
                connection.sendAsynchronous(messageGroup, messageGroup.getDeadlineAbsMillis(messageModule.getAbsMillisTimeSource()));
            }
        }
    /*
        } catch (RuntimeException re) {
            // for debugging only
            re.printStackTrace();
            System.out.println("results.size() "+ results.size());
            for (RetrievalResult result : results) {
                System.out.println(result);
            }
            //rComm.displayDebug();
            System.exit(-1);
            */
    } catch (IOException ioe) {
        ioe.printStackTrace();
    }
}

Example 34 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class ActiveProxyRetrieval method localOp.

@Override
protected void localOp(List<? extends DHTKey> destEntries, OpCommunicator<DHTKey, RetrievalResult> comm) {
    List<ByteBuffer> results;
    results = getStorage().retrieve(getContext(), // entry can act as a key
    destEntries, getRetrievalOptions(), uuid);
    for (int i = 0; i < destEntries.size(); i++) {
        RetrievalResult retrievalResult;
        ByteBuffer result;
        DHTKey entry;
        entry = destEntries.get(i);
        result = results.get(i);
        if (debug) {
            Log.warning("localRetrieval: ", entry);
            System.out.printf("result %s %s\n", result, StringUtil.byteBufferToHexString(result));
        }
        // FUTURE - THIS NEEDS TO GO THROUGH THE PROTOCOL
        // INSTEAD OF PROTOCOL SEMANTICS BEING HANDLED HERE
        Log.fine(entry);
        if (result != null && result != ValueUtil.corruptValue) {
            retrievalResult = new RetrievalResult(entry, OpResult.SUCCEEDED, result);
        } else {
            if (result != ValueUtil.corruptValue) {
                if (retrievalOptions.getWaitMode() != WaitMode.WAIT_FOR || messageModule.getReplicaList(getContext(), entry, OwnerQueryMode.Secondary, RingOwnerQueryOpType.Read).contains(localIPAndPort())) {
                    retrievalResult = new RetrievalResult(entry, OpResult.NO_SUCH_VALUE, null);
                } else {
                    retrievalResult = null;
                }
            } else {
                if (debug) {
                    System.out.println("Returning corrupt result");
                }
                retrievalResult = new RetrievalResult(entry, OpResult.CORRUPT, null);
            }
        }
        if (retrievalResult != null) {
            // retrievalOperation.update((DHTKey)entry, localIPAndPort(), retrievalResult, rComm);
            rComm.sendResult(retrievalResult);
        }
    // Complete operations are removed in bulk by MessageModule.Cleaner
    }
}

Example 35 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method retrieve.

public List<ByteBuffer> retrieve(List<? extends DHTKey> keys, InternalRetrievalOptions options, UUIDBase opUUID) {
    if (retrieveTrigger == null) {
        DHTKey[] _keys;
        ByteBuffer[] _results;
        if (debugVersion) {
            System.out.printf("retrieve internal options: %s\n", options);
        }
        _keys = new DHTKey[keys.size()];
        for (int i = 0; i < _keys.length; i++) {
            _keys[i] = keys.get(i);
        }
        if (_keys.length > 1) {
            _results = _retrieve(_keys, options);
        } else {
            // special case single retrieval
            _results = new ByteBuffer[1];
            _results[0] = _retrieve(_keys[0], options);
        }
        for (int i = 0; i < _results.length; i++) {
            if (parent != null) {
                VersionConstraint vc;
                if (debugParent) {
                    Log.warning("parent != null");
                }
                vc = options.getVersionConstraint();
                if (_results[i] == null) {
                    if (debugParent) {
                        Log.warningf("%x null result. Checking parent %x.", ns, parent.getNamespace());
                    }
                    // If result from this ns is null, look in the parent.
                    _results[i] = parent._retrieve(_keys[i], makeOptionsForNestedRetrieve(options));
                    if (debugParent) {
                        if (_results[i] != null) {
                            Log.warning("Found result in parent");
                        }
                    }
                } else {
                    // Otherwise for the LEAST case, look in the parent to see if it has a better result.
                    if (vc.getMode() == VersionConstraint.Mode.LEAST) {
                        ByteBuffer parentResult;
                        if (debugParent) {
                            Log.warning("Non-null result, but mode LEAST. checking parent");
                        }
                        parentResult = parent._retrieve(_keys[i], makeOptionsForNestedRetrieve(options));
                        if (parentResult != null) {
                            // if the parent had any valid result, then - by virtue of the fact
                            // that all parent versions are < child versions - the parent
                            // result is preferred
                            _results[i] = parentResult;
                            if (_results[i] != null) {
                                Log.warning("Found result in parent");
                            }
                        }
                    }
                }
            }
            if (_results[i] == null && options.getWaitMode() == WaitMode.WAIT_FOR && options.getVersionConstraint().getMax() > curSnapshot) {
                // Note that since we hold the readLock, a write cannot come
                // in while we add the pending wait for.
                addPendingWaitFor(_keys[i], options.getRetrievalOptions(), opUUID);
            }
            if (options.getVerifyIntegrity()) {
                _results[i] = verifyIntegrity(_keys[i], _results[i]);
            }
        }
        return SKImmutableList.copyOf(_results);
    } else {
        return retrieve_nongroupedImpl(keys, options, opUUID);
    }
}