Examples with LongInterval com.ms.silverking.numeric.LongInterval used on opensource projects

Example 1 with LongInterval

use of com.ms.silverking.numeric.LongInterval in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method incomingChecksumTree.

public void incomingChecksumTree(ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    boolean initialSRRSent;
    lastUpdateMillis = SystemTimeSource.instance.absTimeMillis();
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s %s %s estimatedKeys %s", uuid, cp, (connection != null ? connection.getRemoteIPAndPort() : "null connection"), (remoteTree != null ? Integer.toString(remoteTree.estimatedKeys()) : "null remoteTree"));
    }
    if (remoteTree == null) {
        if (debug) {
            Log.warning("null incomingChecksumTree");
        }
        checkForCompletion();
        return;
    }
    if (checksumTreeServer != null) {
        localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
        if (localTree == null) {
            localTree = createEmptyChecksumTree(remoteTree.getRegion());
        }
    } else {
        if (verbose) {
            Log.warningAsync("checksumTreeServer == null\t%s", uuid);
        }
        localTree = createEmptyChecksumTree(remoteTree.getRegion());
    }
    if (debug) {
        System.out.printf("incomingChecksumTree %x\t%s\t%s\t%s\n", namespace, cp, remoteTree.getRegion(), uuid);
    // System.out.println(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (verbose) {
        Log.warningAsyncf("matchResult %s %s", uuid, matchResult.toSummaryString());
    }
    if (debug) {
        System.out.println(matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
		 * Queue<DHTKey> keysToFetchQueue;
		 * 
		 * keysToFetchQueue = keysToFetchMap.get(cp); if (keysToFetchQueue ==
		 * null) { Queue<DHTKey> prev;
		 * 
		 * keysToFetchQueue = new ConcurrentLinkedQueue<>(); prev =
		 * keysToFetchMap.putIfAbsent(cp, keysToFetchQueue); if (prev != null) {
		 * keysToFetchQueue = prev; } } keysToFetchQueue.addAll(keysToFetch);
		 */
    initialSRRSent = false;
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        activeRegionSyncs.put(uuid, this);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        if (!initialSRRSent) {
            initialSRRSent = true;
            sendSyncRetrievalRequest(srr);
        }
    }
    // checkMGQueue();
    if (debug) {
        System.out.println("no more keysToFetch");
    }
    checkForCompletion();
}

Example 2 with LongInterval

use of com.ms.silverking.numeric.LongInterval in project SilverKing by Morgan-Stanley.

the class ChecksumTreeServer method getChecksumTree.

// ////////////////////////////////////////////////////////////////////
// Handle incoming request for a convergence tree
public void getChecksumTree(UUIDBase uuid, ConvergencePoint targetCP, ConvergencePoint sourceCP, MessageGroupConnection connection, byte[] originator, RingRegion region) {
    ChecksumNode root;
    ProtoChecksumTreeMessageGroup pmg;
    int bufferSize;
    ConvergencePoint mixedCP;
    if (verbose || debug) {
        System.out.printf("getChecksumTree %x\t%s\t%s\t%s\t%s\n", ns, targetCP, sourceCP, region, uuid);
    }
    // FUTURE - support both min and max version
    // We create a mixedCP because we need to retrieve target data from the source regions, but
    // we need to use the target data version
    mixedCP = sourceCP.dataVersion(targetCP.getDataVersion());
    root = getRegionChecksumTree_Local(mixedCP, region, new LongInterval(Long.MIN_VALUE, mixedCP.getDataVersion()));
    if (root == null) {
        if (verbose || debug) {
            System.out.printf("null root for %x\t%s\t%s\n", ns, mixedCP, region);
        }
    }
    if (debug) {
        System.out.println("computed:");
        System.out.println(root);
    }
    bufferSize = estimateBufferSize(root);
    pmg = null;
    do {
        try {
            pmg = new ProtoChecksumTreeMessageGroup(uuid, ns, targetCP, originator, root, bufferSize);
        // pmg = new ProtoChecksumTreeMessageGroup(uuid, ns, mixedCP, originator, root, bufferSize);
        } catch (BufferOverflowException bfe) {
            pmg = null;
            if (bufferSize == Integer.MAX_VALUE) {
                throw new RuntimeException("Buffer limit reached");
            } else {
                if (debug) {
                    System.out.printf("raising checksum tree buffer limit %s\n", uuid);
                }
                bufferSize = bufferSize << 1;
                if (bufferSize < 0) {
                    bufferSize = Integer.MAX_VALUE;
                }
            }
        }
    } while (pmg == null);
    try {
        MessageGroup mg;
        mg = pmg.toMessageGroup();
        if (verbose || debug) {
            System.out.printf("Sending checksum tree %s to %s\n", uuid, connection.getRemoteIPAndPort());
        }
        if (debug) {
            System.out.printf("Sending checksum tree at %d %d\n", absMillisTimeSource.absTimeMillis(), mg.getDeadlineAbsMillis(absMillisTimeSource));
        }
        connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe);
    }
}

Example 3 with LongInterval

use of com.ms.silverking.numeric.LongInterval in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method incomingChecksumTree.

// FUTURE - ENSURE THAT NO CONVERGENCE CODE USES THE RECEIVE THREADS TO DO WORK
// USE A WORKER
private void incomingChecksumTree(UUIDBase incomingUUID, ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    outstandingChecksumTreeRequests.remove(incomingUUID);
    checkQueuedChecksumTreeRequests();
    if (remoteTree == null) {
        checkForCompletion();
        return;
    }
    localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s", incomingUUID);
    }
    if (debug) {
        Log.warningAsyncf("incomingChecksumTree %x\t%s\t%s\t%s", ns, cp, remoteTree.getRegion(), incomingUUID);
        Log.warningAsyncf(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (debug) {
        Log.warningAsyncf("%s", matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
        Queue<DHTKey>   keysToFetchQueue;
        
        keysToFetchQueue = keysToFetchMap.get(cp);
        if (keysToFetchQueue == null) {
            Queue<DHTKey>   prev;
            
            keysToFetchQueue = new ConcurrentLinkedQueue<>();
            prev = keysToFetchMap.putIfAbsent(cp, keysToFetchQueue);
            if (prev != null) {
                keysToFetchQueue = prev;
            }
        }
        keysToFetchQueue.addAll(keysToFetch);
        */
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup        mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        convergenceControllers.put(uuid, this);
        sendSyncRetrievalRequest(srr);
    // mg = new ProtoRetrievalMessageGroup(uuid, ns, new InternalRetrievalOptions(retrievalOptions),
    // mgBase.getMyID(), batchKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(connection.getRemoteSocketAddress())));
    /*
            try {
                connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
            } catch (IOException ioe) {
                Log.logErrorWarning(ioe);
            }
            */
    // if (keysToFetchList.size() > 0) {
    // ThreadUtil.sleep(5);
    // }
    }
    checkMGQueue();
    if (debug) {
        Log.warningAsyncf("no more keysToFetch");
    }
    checkForCompletion();
}