Examples with KeyAndVersionChecksum com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum used on opensource projects

Example 1 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class LeafChecksumNode method toString.

@Override
public void toString(StringBuilder sb, int depth) {
    super.toString(sb, depth);
    if (mutability == Mutability.Mutable) {
        for (KeyAndVersionChecksum keyAndVersionChecksum : keyAndVersionChecksums) {
            sb.append(String.format("%s[%s]\n", StringUtil.replicate('\t', depth + 1), keyAndVersionChecksum));
        }
    } else {
        Iterator<KeyAndVersionChecksum> iterator;
        iterator = iterator();
        while (iterator.hasNext()) {
            KeyAndVersionChecksum keyAndVersionChecksum;
            keyAndVersionChecksum = iterator.next();
            sb.append(String.format("%s[%s]\n", StringUtil.replicate('\t', depth + 1), keyAndVersionChecksum));
        }
    }
}

Example 2 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class RegionTreePruner method pruneLeafNode.

private static ChecksumNode pruneLeafNode(LeafChecksumNode node, RingRegion pruneRegion) {
    IntersectionResult iResult;
    RingRegion oRegion;
    // long[]				_kvc;
    List<KeyAndVersionChecksum> kvcList;
    List<KeyAndVersionChecksum> prunedKVCList;
    if (!node.getRegion().overlaps(pruneRegion)) {
        throw new RuntimeException("panic");
    }
    iResult = RingRegion.intersect(node.getRegion(), pruneRegion);
    if (iResult.getIntersectionType() == IntersectionType.wrappedPartial) {
        // FUTURE - handle this
        throw new RuntimeException("wrappedPartial pruning not yet handled");
    }
    oRegion = iResult.getOverlapping().get(0);
    // _kvc = node.getKeyAndVersionChecksumsAsArray();
    // FUTURE - improve the efficiency of this approach
    // This is n log n, and each step is heavy
    // Should be able to do at least log n with very light steps
    kvcList = node.getKeyAndVersionChecksums();
    prunedKVCList = new ArrayList<>(kvcList.size());
    for (KeyAndVersionChecksum kvc : kvcList) {
        if (pruneRegion.contains(KeyUtil.keyToCoordinate(kvc.getKey()))) {
            prunedKVCList.add(kvc);
        }
    }
    return new LeafChecksumNode(oRegion, prunedKVCList);
}

Example 3 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class MatchResult method addList.

private void addList(StringBuilder sb, List<KeyAndVersionChecksum> list, String listName) {
    if (list.size() == 0) {
    // sb.append("\t<empty>\n");
    } else {
        sb.append(listName);
        sb.append('\n');
        for (KeyAndVersionChecksum kvc : list) {
            sb.append('\t');
            sb.append(kvc);
            sb.append('\n');
        }
    }
}

Example 4 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class RegionTreeBuilder method main.

public static void main(String[] args) {
    if (args.length != 4) {
        System.out.println("numKeys keysPerNode regionStart regionEnd");
    } else {
        RegionTreeBuilder rtb;
        int numKeys;
        int keysPerNode;
        long regionStart;
        long regionEnd;
        RingRegion region;
        numKeys = Integer.parseInt(args[0]);
        keysPerNode = Integer.parseInt(args[1]);
        regionStart = Long.parseLong(args[2]);
        regionEnd = Long.parseLong(args[3]);
        region = new RingRegion(regionStart, regionEnd);
        rtb = new RegionTreeBuilder(region, keysPerNode, numKeys);
        for (int i = 0; i < numKeys; i++) {
            rtb.addChecksum(new KeyAndVersionChecksum(KeyUtil.randomRegionKey(region), i));
        }
        System.out.println(rtb);
    }
}

Example 5 with KeyAndVersionChecksum

use of com.ms.silverking.cloud.dht.daemon.storage.KeyAndVersionChecksum in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method incomingChecksumTree.

public void incomingChecksumTree(ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    boolean initialSRRSent;
    lastUpdateMillis = SystemTimeSource.instance.absTimeMillis();
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s %s %s estimatedKeys %s", uuid, cp, (connection != null ? connection.getRemoteIPAndPort() : "null connection"), (remoteTree != null ? Integer.toString(remoteTree.estimatedKeys()) : "null remoteTree"));
    }
    if (remoteTree == null) {
        if (debug) {
            Log.warning("null incomingChecksumTree");
        }
        checkForCompletion();
        return;
    }
    if (checksumTreeServer != null) {
        localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
        if (localTree == null) {
            localTree = createEmptyChecksumTree(remoteTree.getRegion());
        }
    } else {
        if (verbose) {
            Log.warningAsync("checksumTreeServer == null\t%s", uuid);
        }
        localTree = createEmptyChecksumTree(remoteTree.getRegion());
    }
    if (debug) {
        System.out.printf("incomingChecksumTree %x\t%s\t%s\t%s\n", namespace, cp, remoteTree.getRegion(), uuid);
    // System.out.println(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (verbose) {
        Log.warningAsyncf("matchResult %s %s", uuid, matchResult.toSummaryString());
    }
    if (debug) {
        System.out.println(matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
		 * Queue<DHTKey> keysToFetchQueue;
		 * 
		 * keysToFetchQueue = keysToFetchMap.get(cp); if (keysToFetchQueue ==
		 * null) { Queue<DHTKey> prev;
		 * 
		 * keysToFetchQueue = new ConcurrentLinkedQueue<>(); prev =
		 * keysToFetchMap.putIfAbsent(cp, keysToFetchQueue); if (prev != null) {
		 * keysToFetchQueue = prev; } } keysToFetchQueue.addAll(keysToFetch);
		 */
    initialSRRSent = false;
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        activeRegionSyncs.put(uuid, this);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        if (!initialSRRSent) {
            initialSRRSent = true;
            sendSyncRetrievalRequest(srr);
        }
    }
    // checkMGQueue();
    if (debug) {
        System.out.println("no more keysToFetch");
    }
    checkForCompletion();
}