Examples with KeyAndVersionChecksum com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum used on opensource projects

Example 11 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method incomingChecksumTree.

// FUTURE - ENSURE THAT NO CONVERGENCE CODE USES THE RECEIVE THREADS TO DO WORK
// USE A WORKER
private void incomingChecksumTree(UUIDBase incomingUUID, ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    outstandingChecksumTreeRequests.remove(incomingUUID);
    checkQueuedChecksumTreeRequests();
    if (remoteTree == null) {
        checkForCompletion();
        return;
    }
    localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s", incomingUUID);
    }
    if (debug) {
        Log.warningAsyncf("incomingChecksumTree %x\t%s\t%s\t%s", ns, cp, remoteTree.getRegion(), incomingUUID);
        Log.warningAsyncf(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (debug) {
        Log.warningAsyncf("%s", matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
        Queue<DHTKey>   keysToFetchQueue;
        
        keysToFetchQueue = keysToFetchMap.get(cp);
        if (keysToFetchQueue == null) {
            Queue<DHTKey>   prev;
            
            keysToFetchQueue = new ConcurrentLinkedQueue<>();
            prev = keysToFetchMap.putIfAbsent(cp, keysToFetchQueue);
            if (prev != null) {
                keysToFetchQueue = prev;
            }
        }
        keysToFetchQueue.addAll(keysToFetch);
        */
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup        mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        convergenceControllers.put(uuid, this);
        sendSyncRetrievalRequest(srr);
    // mg = new ProtoRetrievalMessageGroup(uuid, ns, new InternalRetrievalOptions(retrievalOptions),
    // mgBase.getMyID(), batchKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(connection.getRemoteSocketAddress())));
    /*
            try {
                connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
            } catch (IOException ioe) {
                Log.logErrorWarning(ioe);
            }
            */
    // if (keysToFetchList.size() > 0) {
    // ThreadUtil.sleep(5);
    // }
    }
    checkMGQueue();
    if (debug) {
        Log.warningAsyncf("no more keysToFetch");
    }
    checkForCompletion();
}

Example 12 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class ProtoChecksumTreeMessageGroup method writeKeyAndVersionChecksums.

private static void writeKeyAndVersionChecksums(ByteBuffer buffer, LeafChecksumNode node) {
    List<KeyAndVersionChecksum> kvcList;
    kvcList = node.getKeyAndVersionChecksums();
    buffer.putInt(kvcList.size());
    for (KeyAndVersionChecksum kvc : kvcList) {
        writeKeyValueChecksum(buffer, kvc);
    }
}

Example 13 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class ProtoChecksumTreeMessageGroup method deserializeKeyAndVersionChecksums.

private static List<KeyAndVersionChecksum> deserializeKeyAndVersionChecksums(ByteBuffer buffer) {
    List<KeyAndVersionChecksum> kvcList;
    int kvcListSize;
    kvcListSize = buffer.getInt();
    kvcList = new ArrayList<>(kvcListSize);
    for (int i = 0; i < kvcListSize; i++) {
        kvcList.add(deserializeKeyAndVersionChecksum(buffer));
    }
    return kvcList;
}

Example 14 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class ProtoChecksumTreeMessageGroup method deserializeKeyAndVersionChecksum.

private static KeyAndVersionChecksum deserializeKeyAndVersionChecksum(ByteBuffer buffer) {
    DHTKey key;
    long versionChecksum;
    key = deserializeKey(buffer);
    versionChecksum = buffer.getLong();
    return new KeyAndVersionChecksum(key, versionChecksum);
}