Examples with VersionConstraint com.ms.silverking.cloud.dht.VersionConstraint used on opensource projects

Example 11 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class ConvergenceController2 method incomingChecksumTree.

// FUTURE - ENSURE THAT NO CONVERGENCE CODE USES THE RECEIVE THREADS TO DO WORK
// USE A WORKER
private void incomingChecksumTree(UUIDBase incomingUUID, ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    outstandingChecksumTreeRequests.remove(incomingUUID);
    checkQueuedChecksumTreeRequests();
    if (remoteTree == null) {
        checkForCompletion();
        return;
    }
    localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s", incomingUUID);
    }
    if (debug) {
        Log.warningAsyncf("incomingChecksumTree %x\t%s\t%s\t%s", ns, cp, remoteTree.getRegion(), incomingUUID);
        Log.warningAsyncf(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (debug) {
        Log.warningAsyncf("%s", matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
        Queue<DHTKey>   keysToFetchQueue;
        
        keysToFetchQueue = keysToFetchMap.get(cp);
        if (keysToFetchQueue == null) {
            Queue<DHTKey>   prev;
            
            keysToFetchQueue = new ConcurrentLinkedQueue<>();
            prev = keysToFetchMap.putIfAbsent(cp, keysToFetchQueue);
            if (prev != null) {
                keysToFetchQueue = prev;
            }
        }
        keysToFetchQueue.addAll(keysToFetch);
        */
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup        mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        convergenceControllers.put(uuid, this);
        sendSyncRetrievalRequest(srr);
    // mg = new ProtoRetrievalMessageGroup(uuid, ns, new InternalRetrievalOptions(retrievalOptions),
    // mgBase.getMyID(), batchKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(connection.getRemoteSocketAddress())));
    /*
            try {
                connection.sendAsynchronous(mg, mg.getDeadlineAbsMillis(absMillisTimeSource));
            } catch (IOException ioe) {
                Log.logErrorWarning(ioe);
            }
            */
    // if (keysToFetchList.size() > 0) {
    // ThreadUtil.sleep(5);
    // }
    }
    checkMGQueue();
    if (debug) {
        Log.warningAsyncf("no more keysToFetch");
    }
    checkForCompletion();
}

Example 12 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class NamespaceStore method retrieve.

public List<ByteBuffer> retrieve(List<? extends DHTKey> keys, InternalRetrievalOptions options, UUIDBase opUUID) {
    if (retrieveTrigger == null) {
        DHTKey[] _keys;
        ByteBuffer[] _results;
        if (debugVersion) {
            System.out.printf("retrieve internal options: %s\n", options);
        }
        _keys = new DHTKey[keys.size()];
        for (int i = 0; i < _keys.length; i++) {
            _keys[i] = keys.get(i);
        }
        if (_keys.length > 1) {
            _results = _retrieve(_keys, options);
        } else {
            // special case single retrieval
            _results = new ByteBuffer[1];
            _results[0] = _retrieve(_keys[0], options);
        }
        for (int i = 0; i < _results.length; i++) {
            if (parent != null) {
                VersionConstraint vc;
                if (debugParent) {
                    Log.warning("parent != null");
                }
                vc = options.getVersionConstraint();
                if (_results[i] == null) {
                    if (debugParent) {
                        Log.warningf("%x null result. Checking parent %x.", ns, parent.getNamespace());
                    }
                    // If result from this ns is null, look in the parent.
                    _results[i] = parent._retrieve(_keys[i], makeOptionsForNestedRetrieve(options));
                    if (debugParent) {
                        if (_results[i] != null) {
                            Log.warning("Found result in parent");
                        }
                    }
                } else {
                    // Otherwise for the LEAST case, look in the parent to see if it has a better result.
                    if (vc.getMode() == VersionConstraint.Mode.LEAST) {
                        ByteBuffer parentResult;
                        if (debugParent) {
                            Log.warning("Non-null result, but mode LEAST. checking parent");
                        }
                        parentResult = parent._retrieve(_keys[i], makeOptionsForNestedRetrieve(options));
                        if (parentResult != null) {
                            // if the parent had any valid result, then - by virtue of the fact
                            // that all parent versions are < child versions - the parent
                            // result is preferred
                            _results[i] = parentResult;
                            if (_results[i] != null) {
                                Log.warning("Found result in parent");
                            }
                        }
                    }
                }
            }
            if (_results[i] == null && options.getWaitMode() == WaitMode.WAIT_FOR && options.getVersionConstraint().getMax() > curSnapshot) {
                // Note that since we hold the readLock, a write cannot come
                // in while we add the pending wait for.
                addPendingWaitFor(_keys[i], options.getRetrievalOptions(), opUUID);
            }
            if (options.getVerifyIntegrity()) {
                _results[i] = verifyIntegrity(_keys[i], _results[i]);
            }
        }
        return SKImmutableList.copyOf(_results);
    } else {
        return retrieve_nongroupedImpl(keys, options, opUUID);
    }
}

Example 13 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class NamespaceStore method getSegmentNumber.

private int getSegmentNumber(DHTKey key, VersionConstraint vc) {
    metaReadLock.lock();
    try {
        int rawSegmentNumber;
        int segmentNumber;
        rawSegmentNumber = valueSegments.get(key);
        if (debugSegments) {
            Log.warning("valueSegments.get " + key + " " + rawSegmentNumber);
        }
        if (rawSegmentNumber < 0) {
            if (rawSegmentNumber != IntCuckooConstants.noSuchValue) {
                OffsetList offsetList;
                offsetList = offsetListStore.getOffsetList(-rawSegmentNumber);
                segmentNumber = offsetList.getOffset(vc, null);
                if (segmentNumber < 0) {
                    if (segmentNumber == OffsetList.NO_MATCH_FOUND) {
                        segmentNumber = IntCuckooConstants.noSuchValue;
                    } else {
                        throw new RuntimeException("Unexpected segmentNumber: " + segmentNumber);
                    }
                }
            } else {
                segmentNumber = IntCuckooConstants.noSuchValue;
            }
        } else {
            segmentNumber = rawSegmentNumber;
        }
        if (debugSegments) {
            Log.warning("offsetList segmentNumber: ", rawSegmentNumber);
        }
        return segmentNumber;
    } finally {
        metaReadLock.unlock();
    }
}

Example 14 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class NamespaceStore method retrieve_nongroupedImpl.

public List<ByteBuffer> retrieve_nongroupedImpl(List<? extends DHTKey> keys, InternalRetrievalOptions options, UUIDBase opUUID) {
    List<ByteBuffer> results;
    KeyAndInteger[] _keys;
    if (debugVersion) {
        System.out.printf("retrieve internal options: %s\n", options);
    }
    results = new ArrayList<>(keys.size());
    for (DHTKey key : keys) {
        // for (KeyAndInteger key : _keys) {
        ByteBuffer result;
        /*
                if (bufferGroup != null) {
                    result = bufferGroup[j];
                } else {
                    result = null;
                }
                */
        if (retrieveTrigger != null) {
            result = retrieveTrigger.retrieve(this, key, options);
        } else {
            result = _retrieve(key, options);
        }
        if (parent != null) {
            VersionConstraint vc;
            if (debugParent) {
                Log.warning("parent != null");
            }
            vc = options.getVersionConstraint();
            if (result == null) {
                if (debugParent) {
                    Log.warningf("%x null result. Checking parent %x.", ns, parent.getNamespace());
                }
                // If result from this ns is null, look in the parent.
                result = parent._retrieve(key, makeOptionsForNestedRetrieve(options));
                if (debugParent) {
                    if (result != null) {
                        Log.warning("Found result in parent");
                    }
                }
            } else {
                // Otherwise for the LEAST case, look in the parent to see if it has a better result.
                if (vc.getMode() == VersionConstraint.Mode.LEAST) {
                    ByteBuffer parentResult;
                    if (debugParent) {
                        Log.warning("Non-null result, but mode LEAST. checking parent");
                    }
                    parentResult = parent._retrieve(key, makeOptionsForNestedRetrieve(options));
                    if (parentResult != null) {
                        // if the parent had any valid result, then - by virtue of the fact
                        // that all parent versions are < child versions - the parent
                        // result is preferred
                        result = parentResult;
                        if (result != null) {
                            Log.warning("Found result in parent");
                        }
                    }
                }
            }
        }
        if (result == null && options.getWaitMode() == WaitMode.WAIT_FOR && options.getVersionConstraint().getMax() > curSnapshot) {
            // Note that since we hold the readLock, a write cannot come
            // in while we add the pending wait for.
            addPendingWaitFor(key, options.getRetrievalOptions(), opUUID);
        }
        if (options.getVerifyIntegrity()) {
            result = verifyIntegrity(key, result);
        }
        results.add(result);
    }
    // }
    return results;
}

Example 15 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class NamespaceStore method makeOptionsForNestedRetrieve.

/**
 * Adjust options for retrieval from parent. I.e. consider minVersion or creationTime as
 * appropriate. Note that there is some looseness to the semantics for versioning
 * that relies on clock synchronization.
 * @param options
 * @return
 */
private InternalRetrievalOptions makeOptionsForNestedRetrieve(InternalRetrievalOptions options) {
    VersionConstraint oldVC;
    // When looking in parents, we only want to see what's present. We don't want
    // to post any pending wait fors as it is impossible - by design - for a child namespace
    // to observe values stored in parents after child creation. Children are clones,
    // and parents/children are distinct after the creation.
    options = options.waitMode(WaitMode.GET);
    oldVC = options.getVersionConstraint();
    switch(nsOptions.getVersionMode()) {
        case SEQUENTIAL:
            throw new RuntimeException("Panic: parent not supported/expected for version mode of SEQUENTIAL");
        case SINGLE_VERSION:
            // to values that were stored before the child creation
            if (debugParent) {
                Log.warning(oldVC);
                Log.warning(options.versionConstraint(oldVC.maxCreationTime(Math.min(oldVC.getMaxCreationTime(), nsProperties.getCreationTime()))).getVersionConstraint());
            }
            return options.versionConstraint(oldVC.maxCreationTime(Math.min(oldVC.getMaxCreationTime(), nsProperties.getCreationTime())));
        case CLIENT_SPECIFIED:
        case SYSTEM_TIME_MILLIS:
        case SYSTEM_TIME_NANOS:
            // that were stored after the child namespace was created
            return options.versionConstraint(oldVC.max(Math.min(oldVC.getMax(), nsProperties.getMinVersion() - 1)));
        default:
            throw new RuntimeException("Panic");
    }
}