Examples with DHTKey com.ms.silverking.cloud.dht.common.DHTKey used on opensource projects

Example 11 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method incomingChecksumTree.

public void incomingChecksumTree(ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    boolean initialSRRSent;
    lastUpdateMillis = SystemTimeSource.instance.absTimeMillis();
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s %s %s estimatedKeys %s", uuid, cp, (connection != null ? connection.getRemoteIPAndPort() : "null connection"), (remoteTree != null ? Integer.toString(remoteTree.estimatedKeys()) : "null remoteTree"));
    }
    if (remoteTree == null) {
        if (debug) {
            Log.warning("null incomingChecksumTree");
        }
        checkForCompletion();
        return;
    }
    if (checksumTreeServer != null) {
        localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
        if (localTree == null) {
            localTree = createEmptyChecksumTree(remoteTree.getRegion());
        }
    } else {
        if (verbose) {
            Log.warningAsync("checksumTreeServer == null\t%s", uuid);
        }
        localTree = createEmptyChecksumTree(remoteTree.getRegion());
    }
    if (debug) {
        System.out.printf("incomingChecksumTree %x\t%s\t%s\t%s\n", namespace, cp, remoteTree.getRegion(), uuid);
    // System.out.println(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (verbose) {
        Log.warningAsyncf("matchResult %s %s", uuid, matchResult.toSummaryString());
    }
    if (debug) {
        System.out.println(matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
		 * Queue<DHTKey> keysToFetchQueue;
		 * 
		 * keysToFetchQueue = keysToFetchMap.get(cp); if (keysToFetchQueue ==
		 * null) { Queue<DHTKey> prev;
		 * 
		 * keysToFetchQueue = new ConcurrentLinkedQueue<>(); prev =
		 * keysToFetchMap.putIfAbsent(cp, keysToFetchQueue); if (prev != null) {
		 * keysToFetchQueue = prev; } } keysToFetchQueue.addAll(keysToFetch);
		 */
    initialSRRSent = false;
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        activeRegionSyncs.put(uuid, this);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        if (!initialSRRSent) {
            initialSRRSent = true;
            sendSyncRetrievalRequest(srr);
        }
    }
    // checkMGQueue();
    if (debug) {
        System.out.println("no more keysToFetch");
    }
    checkForCompletion();
}

Example 12 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class BaseRetrievalOperation method checkForInternalTimeouts.

public Set<IPAndPort> checkForInternalTimeouts(long curTimeMillis, RetrievalVirtualCommunicator rvComm) {
    Set<IPAndPort> timedOutReplicas;
    timedOutReplicas = new HashSet<>();
    if (getMinInternalTimeoutMillis() < curTimeMillis) {
        // while it's being created  FUTURE - make that impossible
        try {
            for (DHTKey key : opKeys()) {
                S entryState;
                entryState = getEntryState(key);
                if (entryState.hasTimedOut(curTimeMillis)) {
                    IPAndPort replica;
                    replica = entryState.currentReplica();
                    if (replica != null && !entryState.prevReplicaSameAsCurrent()) {
                        Log.warning("Non-fatal replica timedOut " + replica + " " + this);
                        timedOutReplicas.add(replica);
                    }
                    tryNextReplica(key, entryState, rvComm);
                }
            }
        } catch (ConcurrentModificationException cme) {
            // FUTURE - This may happen during object creation. Eliminate this possibility in the future.
            // For now, simply ignore. Next check should work.
            Log.warningAsync("Ignoring concurrent modification in BaseRetrievalOperation.checkForInternalTimeouts()");
        }
    }
    return timedOutReplicas;
}

Example 13 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method cleanupPendingWaitFors.

public void cleanupPendingWaitFors() {
    if (debugWaitFor) {
        System.out.println("Cleaning pending waitfors");
    }
    for (Map.Entry<DHTKey, Set<PendingWaitFor>> entry : pendingWaitFors.entrySet()) {
        for (PendingWaitFor pendingWaitFor : entry.getValue()) {
            ActiveProxyRetrieval waiter;
            waiter = activeRetrievals.get(pendingWaitFor.getOpUUID());
            if (waiter == null) {
                if (debugWaitFor) {
                    System.out.printf("No active retrieval for %s\n", pendingWaitFor.getKey());
                } else {
                    System.out.printf("Found active retrieval for %s\n", pendingWaitFor.getKey());
                }
                entry.getValue().remove(pendingWaitFor);
            }
        }
        if (entry.getValue().size() == 0) {
            pendingWaitFors.remove(entry.getKey());
        }
    }
}

Example 14 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method singleReverseSegmentWalk.

public <T extends ValueRetentionState> void singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos) {
    Set<Integer> deletedSegments;
    HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries;
    removedEntries = new HashedSetMap<>();
    deletedSegments = new HashSet<>();
    for (int i = headSegment.getSegmentNumber() - 1; i >= 0; i--) {
        if (segmentExists(i)) {
            Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> result;
            WritableSegmentBase segment;
            CompactionCheckResult ccr;
            try {
                Stopwatch sw;
                sw = new SimpleStopwatch();
                if (nsOptions.getStorageType() == StorageType.RAM) {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                } else {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                // segment = FileSegment.openReadOnly(nsDir, i, nsOptions.getSegmentSize(), nsOptions, SegmentIndexLocation.RAM, SegmentPrereadMode.Preread);
                }
                try {
                    result = segment.singleReverseSegmentWalk(vrp, valueRetentionState, curTimeNanos, ringMaster);
                    ccr = result.getV1();
                } finally {
                    if (nsOptions.getStorageType() == StorageType.FILE) {
                        if (segment.getSegmentNumber() != headSegment.getSegmentNumber()) {
                            ((FileSegment) segment).removeReference();
                        }
                    }
                }
                Log.warningAsyncf("Segment %3d CompactionCheckResult:\t%s", i, ccr.toString());
                if (segment.getSegmentNumber() == headSegment.getSegmentNumber()) {
                    Log.warningAsyncf("Retaining head segment");
                } else {
                    if (ccr.getValidEntries() == 0) {
                        try {
                            recentFileSegments.remove(i);
                            FileSegmentCompactor.delete(nsDir, i);
                            deletedSegments.add(i);
                            if (FileSegment.mapEverything) {
                                ((FileSegment) segment).close();
                            }
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "Failed to delete segment: " + i);
                        }
                    } else if (ccr.getInvalidFraction() >= compactionThreshold) {
                        try {
                            HashedSetMap<DHTKey, Triple<Long, Integer, Long>> segmentRemovedEntries;
                            recentFileSegments.remove(i);
                            segmentRemovedEntries = FileSegmentCompactor.compact(nsDir, i, nsOptions, new RetainedOffsetMapCheck(result.getV2(), result.getV3()));
                            removedEntries.addAll(segmentRemovedEntries);
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "IOException compacting segment: " + i);
                        }
                    }
                }
                sw.stop();
                Log.warningAsyncf("\t\t%d %f", i, sw.getElapsedSeconds());
            } catch (Exception e) {
                Log.logErrorWarning(e, "Skipping segment " + i + " due to Exception");
            }
        }
    }
    updateOffsetLists(deletedSegments, removedEntries);
}

Example 15 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class FileSegmentUtil method main.

public static void main(String[] args) {
    if (args.length != 4) {
        System.err.println("args: <key> <nsDir> <minSegment> <maxSegment>");
    } else {
        try {
            DHTKey key;
            File nsDir;
            int minSegment;
            int maxSegment;
            if (!args[0].equals("*")) {
                key = new StringMD5KeyCreator().createKey(args[0]);
            } else {
                key = null;
            }
            nsDir = new File(args[1]);
            minSegment = Integer.parseInt(args[2]);
            maxSegment = Integer.parseInt(args[3]);
            debugFiles(key, nsDir, minSegment, maxSegment);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}