Examples with VersionConstraint com.ms.silverking.cloud.dht.VersionConstraint used on opensource projects

Example 1 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class PingMultiPong method clientIteration.

public void clientIteration(String pingKey, String pongKeyBase, long version) throws PutException, RetrievalException {
    Set<String> pongKeys;
    pongKeys = new HashSet<>(numServers);
    for (int i = 0; i < numServers; i++) {
        for (int j = 0; j < threadsPerServer; j++) {
            pongKeys.add(pongKeyBase + "." + i + "." + j);
        }
    }
    if (verbose) {
        System.out.println("Put: " + pingKey);
    }
    if (delayMS != 0) {
        if (verbose) {
            System.out.print("Sleeping...");
        }
        ThreadUtil.sleep(delayMS);
        if (verbose) {
            System.out.println("Awake.");
        }
    }
    try {
        syncNSP.put(pingKey, pingKey.getBytes(), defaultPutOptions.version(version));
    } catch (PutException pe) {
        System.out.println("ignoring put exception");
    }
    if (verbose) {
        System.out.println("WaitFor: " + pongKeyBase);
    }
    syncNSP.waitFor(pongKeys, syncNSP.getOptions().getDefaultWaitOptions().versionConstraint(VersionConstraint.exactMatch(version)));
    if (verbose) {
        System.out.println("Received: " + pongKeyBase);
    }
}

Example 2 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class DirectoryInMemorySS method retrieve.

public ByteBuffer retrieve(SSRetrievalOptions options) {
    ByteBuffer rVal;
    VersionConstraint vc;
    SerializedDirectory sd;
    Pair<SSStorageParameters, byte[]> sdp;
    vc = options.getVersionConstraint();
    if (vc.equals(VersionConstraint.greatest)) {
        sd = serializedVersions.get(latestUpdateSP.getVersion());
    } else {
        Map.Entry<Long, SerializedDirectory> entry;
        // For directories, versions must be ascending, max creation time not allowed in vc
        if (vc.getMode().equals(VersionConstraint.Mode.GREATEST)) {
            entry = serializedVersions.floorEntry(vc.getMax());
            if (entry != null && entry.getKey() < vc.getMin()) {
                entry = null;
            }
        } else {
            entry = serializedVersions.ceilingEntry(vc.getMin());
            if (entry != null && entry.getKey() > vc.getMax()) {
                entry = null;
            }
        }
        if (entry != null) {
            sd = entry.getValue();
        } else {
            sd = null;
        }
    }
    if (sd != null) {
        try {
            sdp = sd.readDir();
            rVal = SSUtil.retrievalResultBufferFromValue(sdp.getV2(), StorageParameters.fromSSStorageParameters(sdp.getV1()));
            return rVal;
        } catch (IOException ioe) {
            Log.logErrorWarning(ioe);
        }
    }
    return null;
}

Example 3 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class ProtoRetrievalMessageGroup method getRetrievalOptions.

public static InternalRetrievalOptions getRetrievalOptions(MessageGroup mg) {
    int retrievalWaitByte;
    RetrievalType retrievalType;
    WaitMode waitMode;
    int miscOptionsByte;
    VersionConstraint vc;
    ByteBuffer optionBuffer;
    boolean verifyIntegrity;
    boolean updateSecondariesOnMiss;
    optionBuffer = mg.getBuffers()[optionBufferIndex];
    retrievalWaitByte = optionBuffer.get(RetrievalResponseMessageFormat.retrievalTypeWaitModeOffset);
    // begin retrievalType, waitMode decoding
    // see ProtoRetrievalMessageGroup() for encoding
    retrievalType = EnumValues.retrievalType[retrievalWaitByte >> 4];
    waitMode = EnumValues.waitMode[retrievalWaitByte & 0x0f];
    // end retrievalType, waitMode decoding
    miscOptionsByte = optionBuffer.get(RetrievalResponseMessageFormat.miscOptionsOffset);
    verifyIntegrity = (miscOptionsByte & 0x2) != 0;
    updateSecondariesOnMiss = (miscOptionsByte & 0x1) != 0;
    vc = new VersionConstraint(optionBuffer.getLong(RetrievalResponseMessageFormat.vcMinOffset), optionBuffer.getLong(RetrievalResponseMessageFormat.vcMaxOffset), EnumValues.versionConstraint_Mode[optionBuffer.get(RetrievalResponseMessageFormat.vcModeOffset)], optionBuffer.getLong(RetrievalResponseMessageFormat.vcMaxStorageTimeOffset));
    return new InternalRetrievalOptions(OptionsHelper.newRetrievalOptions(retrievalType, waitMode, vc, updateSecondariesOnMiss, getSecondaryTargets(mg)), verifyIntegrity);
}

Example 4 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method incomingChecksumTree.

public void incomingChecksumTree(ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    boolean initialSRRSent;
    lastUpdateMillis = SystemTimeSource.instance.absTimeMillis();
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s %s %s estimatedKeys %s", uuid, cp, (connection != null ? connection.getRemoteIPAndPort() : "null connection"), (remoteTree != null ? Integer.toString(remoteTree.estimatedKeys()) : "null remoteTree"));
    }
    if (remoteTree == null) {
        if (debug) {
            Log.warning("null incomingChecksumTree");
        }
        checkForCompletion();
        return;
    }
    if (checksumTreeServer != null) {
        localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
        if (localTree == null) {
            localTree = createEmptyChecksumTree(remoteTree.getRegion());
        }
    } else {
        if (verbose) {
            Log.warningAsync("checksumTreeServer == null\t%s", uuid);
        }
        localTree = createEmptyChecksumTree(remoteTree.getRegion());
    }
    if (debug) {
        System.out.printf("incomingChecksumTree %x\t%s\t%s\t%s\n", namespace, cp, remoteTree.getRegion(), uuid);
    // System.out.println(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (verbose) {
        Log.warningAsyncf("matchResult %s %s", uuid, matchResult.toSummaryString());
    }
    if (debug) {
        System.out.println(matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
		 * Queue<DHTKey> keysToFetchQueue;
		 * 
		 * keysToFetchQueue = keysToFetchMap.get(cp); if (keysToFetchQueue ==
		 * null) { Queue<DHTKey> prev;
		 * 
		 * keysToFetchQueue = new ConcurrentLinkedQueue<>(); prev =
		 * keysToFetchMap.putIfAbsent(cp, keysToFetchQueue); if (prev != null) {
		 * keysToFetchQueue = prev; } } keysToFetchQueue.addAll(keysToFetch);
		 */
    initialSRRSent = false;
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        activeRegionSyncs.put(uuid, this);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        if (!initialSRRSent) {
            initialSRRSent = true;
            sendSyncRetrievalRequest(srr);
        }
    }
    // checkMGQueue();
    if (debug) {
        System.out.println("no more keysToFetch");
    }
    checkForCompletion();
}

Example 5 with VersionConstraint

use of com.ms.silverking.cloud.dht.VersionConstraint in project SilverKing by Morgan-Stanley.

the class NamespaceStore method _retrieve.

/**
 * Called when we find a value in the middle of a write operation. We go back to the
 * previously good stored value. Note that we may need to go back more than one value.
 * @param key
 * @param options
 * @return
 */
/*
     * FUTURE - consider removal
    protected ByteBuffer _retrievePrevious(DHTKey key, InternalRetrievalOptions options) {
    	InternalRetrievalOptions curPrevOptions;
    	boolean	found;
    	
    	switch (options.getRetrievalType()) {
    	case VALUE:
    		curPrevOptions = options.retrievalType(RetrievalType.VALUE_AND_META_DATA);
    		break;
    	case VALUE_AND_META_DATA:
    	case META_DATA:
    		curPrevOptions = options;
    		break;
    	case EXISTENCE:
    		curPrevOptions = options.retrievalType(RetrievalType.META_DATA);
    		break;
    	default: throw new RuntimeException("Panic");
    	}
    	//options = options.clipAt();
    	while (true) {
    		ByteBuffer	result;
    		
    		result = _retrieve(key, options);
    		if (result == valueStorageStateInvalidForRead) {
    			
    		} else {
    			return result;
    		}
    	}
    }
    */
/*
    protected ByteBuffer[] _retrieve(KeyAndInteger[] keyGroup, InternalRetrievalOptions options) {
        ByteBuffer[]    result;
        int             segmentNumber;

        segmentNumber = keyGroup[0].getInteger(); // all of this group has the same segmentNumber
        if (debugVersion) {
            System.out.println("retrieve:\t" + keyGroup[0]);
            System.out.println("RetrievalOptions:\t" + options);
        }
        if (segmentNumber == IntCuckooConstants.noSuchValue) {
            return null;
        } else {
            if (headSegment.getSegmentNumber() == segmentNumber) {
                // return getValueEntry(key).retrieve(options);
                if (debugSegments) {
                    Log.warning("Read from head segment");
                }
                result = headSegment.retrieve(keyGroup, options);
                if (debugSegments) {
                    Log.warning("Done read from head segment");
                }
            } else {
                try {
                    AbstractSegment segment;

                    segment = getSegment(segmentNumber);
                    try {
                        if (debugSegments) {
                            Log.warning("Read from file segment");
                        }
                        result = segment.retrieve(keyGroup, options);
                        if (debugSegments) {
                            Log.warning("Done read from file segment");
                            Log.warning("result: " + result);
                        }
                    } finally {
                        if (nsOptions.getStorageType() == StorageType.FILE) {
                            ((FileSegment)segment).removeReference();
                        }
                    }
                } catch (IOException ioe) {
                    Log.logErrorWarning(ioe);
                    return null;
                }
            }
            return result;
        }
    }
    */
protected ByteBuffer _retrieve(DHTKey key, InternalRetrievalOptions options) {
    int segmentNumber;
    VersionConstraint versionConstraint;
    int failedStore;
    if (debugParent) {
        Log.warningf("_retrieve %x %s", ns, key);
    }
    failedStore = 0;
    versionConstraint = options.getVersionConstraint();
    do {
        ByteBuffer result;
        segmentNumber = getSegmentNumber(key, versionConstraint);
        if (debugVersion) {
            System.out.println("retrieve:\t" + key);
            System.out.println("RetrievalOptions:\t" + options);
        }
        if (segmentNumber == IntCuckooConstants.noSuchValue) {
            return null;
        } else {
            readLock.lock();
            try {
                if (headSegment.getSegmentNumber() == segmentNumber) {
                    // return getValueEntry(key).retrieve(options);
                    if (debugSegments) {
                        Log.warning("Read from head segment");
                    }
                    result = retrieve(headSegment, key, options);
                    if (debugSegments) {
                        Log.warning("Done read from head segment");
                    }
                } else {
                    try {
                        AbstractSegment segment;
                        segment = getSegment(segmentNumber, readSegmentPrereadMode);
                        try {
                            if (debugSegments) {
                                Log.warning("Read from file segment");
                            }
                            result = retrieve(segment, key, options);
                            if (debugSegments) {
                                Log.warning("Done read from file segment");
                                Log.warning("result: " + result);
                            }
                        } finally {
                            if (nsOptions.getStorageType() == StorageType.FILE) {
                                if (segment != headSegment) {
                                    ((FileSegment) segment).removeReference();
                                }
                            }
                        }
                    } catch (IOException ioe) {
                        Log.logErrorWarning(ioe);
                        return null;
                    }
                }
            } finally {
                readLock.unlock();
            }
            if (result != null) {
                // Double check that the result is valid
                if (debug) {
                    System.out.printf("tpc %d\n", CCSSUtil.getStorageState(MetaDataUtil.getCCSS(result, 0)));
                    System.out.printf("tpc %s\n", StorageProtocolUtil.storageStateValidForRead(nsOptions.getConsistencyProtocol(), CCSSUtil.getStorageState(MetaDataUtil.getCCSS(result, 0))));
                }
                // FUTURE - this is a temporary workaround until the versioned storage is overhauled
                if (!StorageProtocolUtil.storageStateValidForRead(nsOptions.getConsistencyProtocol(), MetaDataUtil.getStorageState(result, 0))) {
                    // CCSSUtil.getStorageState(MetaDataUtil.getCCSS(result, 0)));
                    switch(versionConstraint.getMode()) {
                        case GREATEST:
                            long newMaxVersion;
                            newMaxVersion = MetaDataUtil.getVersion(result, 0) - 1;
                            if (newMaxVersion < versionConstraint.getMin()) {
                                return null;
                            }
                            versionConstraint = versionConstraint.max(newMaxVersion);
                            break;
                        case LEAST:
                            long newMinVersion;
                            newMinVersion = MetaDataUtil.getVersion(result, 0) + 1;
                            if (newMinVersion > versionConstraint.getMax()) {
                                return null;
                            }
                            versionConstraint = versionConstraint.min(newMinVersion);
                            break;
                        default:
                            throw new RuntimeException("Panic");
                    }
                    options = options.versionConstraint(versionConstraint);
                } else {
                    return result;
                }
            } else {
                return null;
            }
        }
    } while (failedStore++ < maxFailedStores);
    Log.warning("maxFailedStores exceeded");
    return null;
}