Examples with RetrievalOptions com.ms.silverking.cloud.dht.RetrievalOptions used on opensource projects

Example 1 with RetrievalOptions

use of com.ms.silverking.cloud.dht.RetrievalOptions in project SilverKing by Morgan-Stanley.

the class SilverKingClient method retrieveAllValuesForKey.

private Map<String, ? extends StoredValue<byte[]>> retrieveAllValuesForKey(String key) throws RetrievalException {
    RetrievalOptions retrievalOptions;
    Map<String, StoredValue<byte[]>> keyValues;
    keyValues = new HashMap<>();
    retrievalOptions = syncNSP.getOptions().getDefaultGetOptions().retrievalType(RetrievalType.META_DATA).nonExistenceResponse(NonExistenceResponse.NULL_VALUE).versionConstraint(VersionConstraint.greatest).returnInvalidations(true);
    try {
        do {
            StoredValue<byte[]> storedValue;
            // System.out.printf("%s\t%s\n", key, retrievalOptions);
            storedValue = syncNSP.retrieve(key, retrievalOptions);
            if (storedValue != null) {
                String keyAndVersion;
                keyAndVersion = String.format("%s %d %d %s", key, storedValue.getVersion(), storedValue.getCreationTime().inNanos(), storedValue.getCreationTime().toDateString());
                keyValues.put(keyAndVersion, storedValue);
                // System.out.printf("%d\t%d\n", storedValue.getMetaData().getVersion(), storedValue.getMetaData().getCreationTime().inNanos());
                // retrievalOptions = retrievalOptions.versionConstraint(retrievalOptions.getVersionConstraint().maxCreationTime(storedValue.getCreationTime().inNanos() - 1));
                retrievalOptions = retrievalOptions.versionConstraint(retrievalOptions.getVersionConstraint().maxBelowOrEqual(storedValue.getVersion() - 1));
            // ThreadUtil.sleep(1000);
            } else {
                break;
            }
        } while (true);
    } catch (RetrievalException re) {
        displayRetrievalExceptionDetails(re);
        throw re;
    }
    return keyValues;
}

Example 2 with RetrievalOptions

use of com.ms.silverking.cloud.dht.RetrievalOptions in project SilverKing by Morgan-Stanley.

the class SilverKingClient method doRetrieve.

private void doRetrieve(String[] args, RetrievalType retrievalType) throws OperationException, IOException {
    Map<String, ? extends StoredValue<byte[]>> storedValues;
    Set<String> keys;
    RetrievalOptions retrievalOptions;
    if (args[0].startsWith("{")) {
        if (!args[0].endsWith("}")) {
            err.printf("retrievalOptions missing closing }\n");
            return;
        } else {
            String[] newArgs;
            newArgs = new String[args.length - 1];
            System.arraycopy(args, 1, newArgs, 0, newArgs.length);
            retrievalOptions = ((RetrievalOptions) ObjectDefParser2.parse(RetrievalOptions.class, // .retrievalType(retrievalType);
            args[0].substring(1, args[0].length() - 1)));
            retrievalType = retrievalOptions.getRetrievalType();
            if (verbose) {
                out.printf("retrievalOptions: %s\n", retrievalOptions);
            }
            args = newArgs;
        }
    } else {
        retrievalOptions = syncNSP.getNamespace().getOptions().getDefaultGetOptions();
    }
    keys = retrievalKeySet(args);
    storedValues = retrieve(args, retrievalOptions);
    if (retrievalType.hasValue()) {
        displayValueMap(keys, storedValues);
    }
    if (retrievalType == RetrievalType.META_DATA) {
        displayMetaDataMap(keys, storedValues);
    } else if (retrievalType == RetrievalType.EXISTENCE) {
        displayExistence(keys, storedValues);
    }
}

Example 3 with RetrievalOptions

use of com.ms.silverking.cloud.dht.RetrievalOptions in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method incomingChecksumTree.

public void incomingChecksumTree(ConvergencePoint cp, ChecksumNode remoteTree, MessageGroupConnection connection) {
    ChecksumNode localTree;
    MatchResult matchResult;
    Set<DHTKey> keysToFetch;
    List<DHTKey> keysToFetchList;
    boolean initialSRRSent;
    lastUpdateMillis = SystemTimeSource.instance.absTimeMillis();
    if (verbose) {
        Log.warningAsyncf("incomingChecksumTree %s %s %s estimatedKeys %s", uuid, cp, (connection != null ? connection.getRemoteIPAndPort() : "null connection"), (remoteTree != null ? Integer.toString(remoteTree.estimatedKeys()) : "null remoteTree"));
    }
    if (remoteTree == null) {
        if (debug) {
            Log.warning("null incomingChecksumTree");
        }
        checkForCompletion();
        return;
    }
    if (checksumTreeServer != null) {
        localTree = checksumTreeServer.getRegionChecksumTree_Local(cp, remoteTree.getRegion(), new LongInterval(Long.MIN_VALUE, cp.getDataVersion()));
        if (localTree == null) {
            localTree = createEmptyChecksumTree(remoteTree.getRegion());
        }
    } else {
        if (verbose) {
            Log.warningAsync("checksumTreeServer == null\t%s", uuid);
        }
        localTree = createEmptyChecksumTree(remoteTree.getRegion());
    }
    if (debug) {
        System.out.printf("incomingChecksumTree %x\t%s\t%s\t%s\n", namespace, cp, remoteTree.getRegion(), uuid);
    // System.out.println(remoteTree + "\n\nlocalTree\n" + localTree);
    }
    if (localTree == null) {
        checkForCompletion();
        return;
    }
    try {
        matchResult = TreeMatcher.match(localTree, remoteTree);
    } catch (RuntimeException re) {
        System.err.println(localTree);
        System.err.println();
        System.err.println(connection);
        System.err.println(remoteTree);
        throw re;
    }
    if (verbose) {
        Log.warningAsyncf("matchResult %s %s", uuid, matchResult.toSummaryString());
    }
    if (debug) {
        System.out.println(matchResult);
    }
    keysToFetch = new HashSet<>();
    for (KeyAndVersionChecksum kvc : matchResult.getDestNotInSource()) {
        if (debug) {
            System.out.printf("Adding destNotInSource %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    for (KeyAndVersionChecksum kvc : matchResult.getChecksumMismatch()) {
        if (debug) {
            System.out.printf("Adding checksumMismatch %s\n", kvc.getKey());
        }
        keysToFetch.add(kvc.getKey());
    }
    /*
		 * Queue<DHTKey> keysToFetchQueue;
		 * 
		 * keysToFetchQueue = keysToFetchMap.get(cp); if (keysToFetchQueue ==
		 * null) { Queue<DHTKey> prev;
		 * 
		 * keysToFetchQueue = new ConcurrentLinkedQueue<>(); prev =
		 * keysToFetchMap.putIfAbsent(cp, keysToFetchQueue); if (prev != null) {
		 * keysToFetchQueue = prev; } } keysToFetchQueue.addAll(keysToFetch);
		 */
    initialSRRSent = false;
    keysToFetchList = new LinkedList<>(keysToFetch);
    while (keysToFetchList.size() > 0) {
        Set<DHTKey> batchKeys;
        int batchSize;
        RetrievalOptions retrievalOptions;
        UUIDBase uuid;
        // MessageGroup mg;
        SyncRetrievalRequest srr;
        // batchKeys = new HashSet<>(retrievalBatchSize);
        batchKeys = new ConcurrentSkipListSet<DHTKey>();
        batchSize = 0;
        while (keysToFetchList.size() > 0 && batchSize < retrievalBatchSize) {
            batchKeys.add(keysToFetchList.remove(0));
            ++batchSize;
        }
        // FUTURE - could consider sending SourceNotInDest
        retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(cp.getDataVersion()));
        // new VersionConstraint(Long.MIN_VALUE + 1, version, Mode.NEWEST));
        uuid = UUIDBase.random();
        srr = new SyncRetrievalRequest(uuid, batchKeys, cp.getDataVersion(), connection);
        activeRegionSyncs.put(uuid, this);
        outstandingSyncRetrievalRequests.put(uuid, srr);
        if (!initialSRRSent) {
            initialSRRSent = true;
            sendSyncRetrievalRequest(srr);
        }
    }
    // checkMGQueue();
    if (debug) {
        System.out.println("no more keysToFetch");
    }
    checkForCompletion();
}

Example 4 with RetrievalOptions

use of com.ms.silverking.cloud.dht.RetrievalOptions in project SilverKing by Morgan-Stanley.

the class ActiveRegionSync method sendSyncRetrievalRequest.

private void sendSyncRetrievalRequest(SyncRetrievalRequest srr) {
    MessageGroup mg;
    RetrievalOptions retrievalOptions;
    Log.warningAsyncf("ars %s send srr %s", uuid, srr.getUUID());
    inprocessSyncRetrievalRequests.add(srr.getUUID());
    retrievalOptions = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE_AND_META_DATA, WaitMode.GET, checksumVersionConstraint(srr.dataVersion));
    mg = new ProtoRetrievalMessageGroup(srr.uuid, namespace, new InternalRetrievalOptions(retrievalOptions), mgBase.getMyID(), srr.outstandingKeys, convergenceRelativeDeadlineMillis).toMessageGroup();
    // outgoingMessages.add(new OutgoingMessage(mg, new IPAndPort(srr.connection.getRemoteSocketAddress())));
    mgBase.send(mg, srr.connection.getRemoteIPAndPort());
}

Example 5 with RetrievalOptions

use of com.ms.silverking.cloud.dht.RetrievalOptions in project SilverKing by Morgan-Stanley.

the class NamespaceStore method checkForDuplicateStore.

/*
     * 
     * The PendingPut implementation is currently commented out. It's utility is yet to be determined. A more pressing optimization is to 
     * allow a per-thread NamespaceStore so that RAM stores may be lockless.
    
    private static class PendingPut {
    	final List<StorageValueAndParameters> values;
    	final byte[] userData;
    	final KeyedOpResultListener resultListener;
    	
    	PendingPut(List<StorageValueAndParameters> values, byte[] userData, KeyedOpResultListener resultListener) {
    		this.values = values;
    		this.userData = userData;
    		this.resultListener = resultListener;
    	}
    }
    
    private final LightLinkedBlockingQueue<PendingPut>	pendingPuts = new LightLinkedBlockingQueue<>();
    private volatile	boolean	lockedWillPoll;
    
    public void put(List<StorageValueAndParameters> _values, byte[] _userData, KeyedOpResultListener _resultListener) {
        boolean	locked;
        PendingPut	storedPendingPut;

        storedPendingPut = null;
        //LWTThreadUtil.setBlocked();
        locked = writeLock.tryLock();
        if (!locked) {
        	try {
        		storedPendingPut = new PendingPut(_values, _userData, _resultListener);
				pendingPuts.put(storedPendingPut);
        		if (!lockedWillPoll) {
        			writeLock.lock();
        			locked = true;
        		}
			} catch (InterruptedException e) {
				throw new RuntimeException("Panic");
			}
        }
        if (locked) {
	        try {
	        	List<StorageValueAndParameters> values;
	        	byte[] userData;
	        	KeyedOpResultListener resultListener;

	        	lockedWillPoll = true;
	        	if (storedPendingPut != null) {
		        	boolean	prevPendingPutFound;
		        	
	        		prevPendingPutFound = pendingPuts.remove(storedPendingPut);
	        		if (!prevPendingPutFound) {
			            PendingPut	pendingPut;
			            
	    	        	lockedWillPoll = false;
		        		pendingPut = pendingPuts.poll();
			            if (pendingPut == null) {
		        			return;
			            } else {
				        	_values = pendingPut.values;
				        	_userData = pendingPut.userData;
				        	_resultListener = pendingPut.resultListener;
			            }
	        		}
	        	}
	        	values = _values;
	        	userData = _userData;
	        	resultListener = _resultListener;
	        	do {
		            PendingPut	pendingPut;
		            
	        		lockedWillPoll = true;
	        		__put(values, userData, resultListener);

	        		lockedWillPoll = false;
	        		pendingPut = pendingPuts.poll();
		            if (pendingPut == null) {
			            writeLock.unlock();
			            pendingPut = pendingPuts.poll();
			            if (pendingPut == null) {
			            	break;
			            } else {
			            	writeLock.lock();
			            }
		            } else {
			        	values = pendingPut.values;
			        	userData = pendingPut.userData;
			        	resultListener = pendingPut.resultListener;
		            }		            	
	            } while (true);
	        } finally {
	        	if (((ReentrantReadWriteLock)rwLock).isWriteLockedByCurrentThread()) {
	        		writeLock.unlock();
	        	}
	            //LWTThreadUtil.setNonBlocked();
	        }
        }
    }

    // lock must be held
    private void __put(List<StorageValueAndParameters> values, byte[] userData, KeyedOpResultListener resultListener) {
        NamespaceVersionMode    nsVersionMode;
        Set<Waiter> triggeredWaitFors;

        triggeredWaitFors = null;
        nsVersionMode = nsOptions.getVersionMode();
        // System.out.printf("NamespaceStore.put() group size: %d\n", values.size());
        for (StorageValueAndParameters value : values) {
            OpResult storageResult;

            storageResult = _put(value.getKey(), value.getValue(), value, userData, nsVersionMode);
            resultListener.sendResult(value.getKey(), storageResult);
            if (storageResult == OpResult.SUCCEEDED) {
                Set<Waiter> _triggeredWaitFors;

                _triggeredWaitFors = checkPendingWaitFors(value.getKey());
                if (_triggeredWaitFors != null) {
                    if (triggeredWaitFors == null) {
                        triggeredWaitFors = new HashSet<>();
                    }
                    triggeredWaitFors.addAll(_triggeredWaitFors);
                }
            }
        }
        if (triggeredWaitFors != null) {
        	writeLock.unlock();
        	try {
        		handleTriggeredWaitFors(triggeredWaitFors);
        	} finally {
            	writeLock.lock();
        	}
        }
    }
    */
/**
 * Checks to see if this put() should be allowed to proceed on the basis that it is a duplicate of a previous
 * storage operation.
 *
 * @param key
 * @param value
 * @param storageParams
 * @param userData
 * @return
 */
private SegmentStorageResult checkForDuplicateStore(DHTKey key, ByteBuffer value, StorageParameters storageParams, byte[] userData) {
    RetrievalOptions options;
    ByteBuffer result;
    int debug = 0;
    // this comparison isn't returning invalid version like it should for some cases
    options = OptionsHelper.newRetrievalOptions(RetrievalType.VALUE, WaitMode.GET, VersionConstraint.exactMatch(storageParams.getVersion()), NonExistenceResponse.EXCEPTION, true);
    result = _retrieve(key, options);
    if (result == null) {
        debug = -1;
        // previous storage operation is not complete
        return SegmentStorageResult.previousStoreIncomplete;
    } else {
        if (!StorageProtocolUtil.storageStateValidForRead(nsOptions.getConsistencyProtocol(), MetaDataUtil.getStorageState(result, 0))) {
            debug = -2;
            // previous storage operation is not complete
            return SegmentStorageResult.previousStoreIncomplete;
        } else {
            // FUTURE - allow for no mutation detection on a per-namespace basis?
            if (storageParams.getChecksumType() == MetaDataUtil.getChecksumType(result, 0)) {
                if (storageParams.getChecksumType() != ChecksumType.NONE) {
                    if (Arrays.equals(MetaDataUtil.getChecksum(result, 0), storageParams.getChecksum())) {
                        debug = -10;
                        return SegmentStorageResult.duplicateStore;
                    } else {
                        debug = 1;
                        return SegmentStorageResult.mutation;
                    }
                } else {
                    if (BufferUtil.equals(result, MetaDataUtil.getDataOffset(result, 0), value, 0, value.limit())) {
                        debug = -11;
                        return SegmentStorageResult.duplicateStore;
                    } else {
                        debug = 2;
                        return SegmentStorageResult.mutation;
                    }
                }
            } else {
                debug = 3;
                return SegmentStorageResult.mutation;
            }
        }
    }
}