Examples with InternalRetrievalOptions com.ms.silverking.cloud.dht.common.InternalRetrievalOptions used on opensource projects

Example 1 with InternalRetrievalOptions

use of com.ms.silverking.cloud.dht.common.InternalRetrievalOptions in project SilverKing by Morgan-Stanley.

the class AsyncRetrievalOperationImpl method createProtoRetrievalMG.

private ProtoRetrievalMessageGroup createProtoRetrievalMG(MessageEstimate estimate, boolean verifyIntegrity) {
    OperationUUID opUUID;
    ConcurrentMap<DHTKey, List<ActiveKeyedOperationResultListener<MessageGroupRetrievalResponseEntry>>> newMap;
    KeyedMessageEstimate keyedMessageEstimate;
    int relDeadline;
    keyedMessageEstimate = (KeyedMessageEstimate) estimate;
    opUUID = activeRetrievalListeners.newOpUUID();
    relDeadline = operation.getTimeoutController().getMaxRelativeTimeoutMillis(this);
    if (debugShortTimeout) {
        if (relDeadline < shortTimeoutLimit) {
            Log.warning("short relDeadline: " + relDeadline);
            // Log.warning(timeoutParameters.computeTimeout(keyedMessageEstimate.getNumKeys()));
            Log.warning(timeoutState.getCurRelTimeoutMillis());
        // throw new RuntimeException();
        }
    }
    return new ProtoRetrievalMessageGroup(opUUID, context.contextAsLong(), new InternalRetrievalOptions(retrievalOperation.retrievalOptions(), verifyIntegrity), originator, keyedMessageEstimate.getNumKeys(), relDeadline, retrievalOperation.retrievalOptions().getForwardingMode());
}

Example 2 with InternalRetrievalOptions

use of com.ms.silverking.cloud.dht.common.InternalRetrievalOptions in project SilverKing by Morgan-Stanley.

the class ProtoRetrievalMessageGroup method getRetrievalOptions.

public static InternalRetrievalOptions getRetrievalOptions(MessageGroup mg) {
    int retrievalWaitByte;
    RetrievalType retrievalType;
    WaitMode waitMode;
    int miscOptionsByte;
    VersionConstraint vc;
    ByteBuffer optionBuffer;
    boolean verifyIntegrity;
    boolean updateSecondariesOnMiss;
    optionBuffer = mg.getBuffers()[optionBufferIndex];
    retrievalWaitByte = optionBuffer.get(RetrievalResponseMessageFormat.retrievalTypeWaitModeOffset);
    // begin retrievalType, waitMode decoding
    // see ProtoRetrievalMessageGroup() for encoding
    retrievalType = EnumValues.retrievalType[retrievalWaitByte >> 4];
    waitMode = EnumValues.waitMode[retrievalWaitByte & 0x0f];
    // end retrievalType, waitMode decoding
    miscOptionsByte = optionBuffer.get(RetrievalResponseMessageFormat.miscOptionsOffset);
    verifyIntegrity = (miscOptionsByte & 0x2) != 0;
    updateSecondariesOnMiss = (miscOptionsByte & 0x1) != 0;
    vc = new VersionConstraint(optionBuffer.getLong(RetrievalResponseMessageFormat.vcMinOffset), optionBuffer.getLong(RetrievalResponseMessageFormat.vcMaxOffset), EnumValues.versionConstraint_Mode[optionBuffer.get(RetrievalResponseMessageFormat.vcModeOffset)], optionBuffer.getLong(RetrievalResponseMessageFormat.vcMaxStorageTimeOffset));
    return new InternalRetrievalOptions(OptionsHelper.newRetrievalOptions(retrievalType, waitMode, vc, updateSecondariesOnMiss, getSecondaryTargets(mg)), verifyIntegrity);
}

Example 3 with InternalRetrievalOptions

use of com.ms.silverking.cloud.dht.common.InternalRetrievalOptions in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method sendSyncRetrievalRequest.

private void sendSyncRetrievalRequest(SyncRetrievalRequest srr) {
    MessageGroup mg;
    RetrievalOptions retrievalOptions;
    Log.warningAsyncf("ars %s send srr %s", uuid, srr.getUUID());
    inprocessSyncRetrievalRequests.add(srr.getUUID());
    retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(srr.dataVersion));
    mg = new ProtoRetrievalMessageGroup(srr.uuid, namespace, new InternalRetrievalOptions(retrievalOptions), mgBase.getMyID(), srr.outstandingKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(srr.connection.getRemoteSocketAddress())));
    mgBase.send(mg, srr.connection.getRemoteIPAndPort());
}

Example 4 with InternalRetrievalOptions

use of com.ms.silverking.cloud.dht.common.InternalRetrievalOptions in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method incomingChecksumTree.

// FUTURE - ENSURE THAT NO CONVERGENCE CODE USES THE RECEIVE THREADS TO DO WORK
// USE A WORKER
private void incomingChecksumTree(UUIDBase incomingUUID, ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    outstandingChecksumTreeRequests.remove(incomingUUID);
    checkQueuedChecksumTreeRequests();
    if (remoteTree == null) {
        checkForCompletion();
        return;
    }
    localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s", incomingUUID);
    }
    if (debug) {
        Log.warningAsyncf("incomingChecksumTree %x\t%s\t%s\t%s", ns, cp, remoteTree.getRegion(), incomingUUID);
        Log.warningAsyncf(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (debug) {
        Log.warningAsyncf("%s", matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
        Queue<DHTKey>   keysToFetchQueue;
        
        keysToFetchQueue = keysToFetchMap.get(cp);
        if (keysToFetchQueue == null) {
            Queue<DHTKey>   prev;
            
            keysToFetchQueue = new ConcurrentLinkedQueue<>();
            prev = keysToFetchMap.putIfAbsent(cp, keysToFetchQueue);
            if (prev != null) {
                keysToFetchQueue = prev;
            }
        }
        keysToFetchQueue.addAll(keysToFetch);
        */
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup        mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        convergenceControllers.put(uuid, this);
        sendSyncRetrievalRequest(srr);
    // mg = new ProtoRetrievalMessageGroup(uuid, ns, new InternalRetrievalOptions(retrievalOptions),
    // mgBase.getMyID(), batchKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(connection.getRemoteSocketAddress())));
    /*
            try {
                connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
            } catch (IOException ioe) {
                Log.logErrorWarning(ioe);
            }
            */
    // if (keysToFetchList.size() > 0) {
    // ThreadUtil.sleep(5);
    // }
    }
    checkMGQueue();
    if (debug) {
        Log.warningAsyncf("no more keysToFetch");
    }
    checkForCompletion();
}

Example 5 with InternalRetrievalOptions

use of com.ms.silverking.cloud.dht.common.InternalRetrievalOptions in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method sendSyncRetrievalRequest.

private void sendSyncRetrievalRequest(SyncRetrievalRequest srr) {
    MessageGroup mg;
    RetrievalOptions retrievalOptions;
    retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(srr.dataVersion));
    mg = new ProtoRetrievalMessageGroup(srr.uuid, ns, new InternalRetrievalOptions(retrievalOptions), mgBase.getMyID(), srr.outstandingKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(srr.connection.getRemoteSocketAddress())));
}