Examples with DHTKey com.ms.silverking.cloud.dht.common.DHTKey used on opensource projects

Example 36 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method retrieve_nongroupedImpl.

public List<ByteBuffer> retrieve_nongroupedImpl(List<? extends DHTKey> keys, InternalRetrievalOptions options, UUIDBase opUUID) {
    List<ByteBuffer> results;
    KeyAndInteger[] _keys;
    if (debugVersion) {
        System.out.printf("retrieve internal options: %s\n", options);
    }
    results = new ArrayList<>(keys.size());
    for (DHTKey key : keys) {
        // for (KeyAndInteger key : _keys) {
        ByteBuffer result;
        /*
                if (bufferGroup != null) {
                    result = bufferGroup[j];
                } else {
                    result = null;
                }
                */
        if (retrieveTrigger != null) {
            result = retrieveTrigger.retrieve(this, key, options);
        } else {
            result = _retrieve(key, options);
        }
        if (parent != null) {
            VersionConstraint vc;
            if (debugParent) {
                Log.warning("parent != null");
            }
            vc = options.getVersionConstraint();
            if (result == null) {
                if (debugParent) {
                    Log.warningf("%x null result. Checking parent %x.", ns, parent.getNamespace());
                }
                // If result from this ns is null, look in the parent.
                result = parent._retrieve(key, makeOptionsForNestedRetrieve(options));
                if (debugParent) {
                    if (result != null) {
                        Log.warning("Found result in parent");
                    }
                }
            } else {
                // Otherwise for the LEAST case, look in the parent to see if it has a better result.
                if (vc.getMode() == VersionConstraint.Mode.LEAST) {
                    ByteBuffer parentResult;
                    if (debugParent) {
                        Log.warning("Non-null result, but mode LEAST. checking parent");
                    }
                    parentResult = parent._retrieve(key, makeOptionsForNestedRetrieve(options));
                    if (parentResult != null) {
                        // if the parent had any valid result, then - by virtue of the fact
                        // that all parent versions are < child versions - the parent
                        // result is preferred
                        result = parentResult;
                        if (result != null) {
                            Log.warning("Found result in parent");
                        }
                    }
                }
            }
        }
        if (result == null && options.getWaitMode() == WaitMode.WAIT_FOR && options.getVersionConstraint().getMax() > curSnapshot) {
            // Note that since we hold the readLock, a write cannot come
            // in while we add the pending wait for.
            addPendingWaitFor(key, options.getRetrievalOptions(), opUUID);
        }
        if (options.getVerifyIntegrity()) {
            result = verifyIntegrity(key, result);
        }
        results.add(result);
    }
    // }
    return results;
}

Example 37 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method putUpdate.

public List<OpResult> putUpdate(List<? extends DHTKey> updates, long version) {
    List<OpResult> results;
    Set<Waiter> triggeredWaitFors;
    triggeredWaitFors = null;
    results = new ArrayList<>(updates.size());
    writeLock.lock();
    try {
        for (DHTKey update : updates) {
            MessageGroupKeyOrdinalEntry entry;
            OpResult result;
            entry = (MessageGroupKeyOrdinalEntry) update;
            result = _putUpdate(entry, version, entry.getOrdinal());
            results.add(result);
            if (result == OpResult.SUCCEEDED && StorageProtocolUtil.storageStateValidForRead(nsOptions.getConsistencyProtocol(), entry.getOrdinal())) {
                Set<Waiter> _triggeredWaitFors;
                _triggeredWaitFors = checkPendingWaitFors(update);
                if (_triggeredWaitFors != null) {
                    if (triggeredWaitFors == null) {
                        triggeredWaitFors = new HashSet<>();
                    }
                    triggeredWaitFors.addAll(_triggeredWaitFors);
                }
            }
        }
    } finally {
        writeLock.unlock();
    }
    if (triggeredWaitFors != null) {
        handleTriggeredWaitFors(triggeredWaitFors);
    }
    return results;
}

Example 38 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class NamespaceStore method updateOffsetLists.

private void updateOffsetLists(Set<Integer> deletedSegments, HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries) {
    RAMOffsetListStore ols;
    ols = (RAMOffsetListStore) offsetListStore;
    if (deletedSegments.size() > 0) {
        Set<DHTKey> singleKeysInDeletedSegments;
        for (int i = 1; i <= ols.getNumLists(); i++) {
            // offset list indexing is 1-based
            RAMOffsetList ol;
            ol = (RAMOffsetList) ols.getOffsetList(i);
            ol.removeEntriesByValue(deletedSegments);
        }
        singleKeysInDeletedSegments = new HashSet<>();
        for (DHTKeyIntEntry keyAndSegment : valueSegments) {
            if (keyAndSegment.getValue() >= 0 && deletedSegments.contains(keyAndSegment.getValue())) {
                singleKeysInDeletedSegments.add(keyAndSegment.getKey());
            }
        }
        for (DHTKey singleKeyToRemove : singleKeysInDeletedSegments) {
            valueSegments.remove(singleKeyToRemove);
        }
    }
    for (DHTKey key : removedEntries.getKeys()) {
        int rawSegmentNumber;
        rawSegmentNumber = valueSegments.get(key);
        if (rawSegmentNumber < 0) {
            if (rawSegmentNumber != IntCuckooConstants.noSuchValue) {
                RAMOffsetList ol;
                ol = (RAMOffsetList) ols.getOffsetList(-rawSegmentNumber);
                ol.removeEntriesByMatch(removedEntries.getSet(key));
            } else {
            // No action required
            }
        } else {
            long[] versionAndStorageTime;
            long creationTime;
            // only one should exist
            versionAndStorageTime = segmentOldestVersion(rawSegmentNumber, key);
            if (removedEntries.getSet(key).size() != 1) {
                Log.warningAsyncf("Unexpected removedEntries.getSet(key).size() != 1");
            }
            creationTime = nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS ? versionAndStorageTime[1] : 0;
            if (removedEntries.getSet(key).contains(new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime))) {
                valueSegments.remove(key);
            } else {
                Log.warningAsyncf("Unexpected !removedEntries.getSet(key).contains(new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime))");
                Log.warningAsyncf("%s", new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime));
            }
        }
    }
}

Example 39 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class WritableSegmentBase method singleReverseSegmentWalk.

public <T extends ValueRetentionState> Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos, NodeRingMaster2 ringMaster) {
    int numRetained;
    int numDiscarded;
    Set<Integer> retainedOffsets;
    Set<Integer> discardedOffsets;
    numRetained = 0;
    numDiscarded = 0;
    retainedOffsets = new HashSet<>();
    discardedOffsets = new HashSet<>();
    // The outer loop loops through the keys first.
    for (DHTKeyIntEntry entry : keyToOffset) {
        int rawOffset;
        List<Triple<Integer, Long, Long>> offsetVersionAndCreationTimeList;
        rawOffset = entry.getValue();
        if (rawOffset >= 0) {
            long version;
            long creationTime;
            version = getVersion(rawOffset);
            creationTime = getCreationTime(rawOffset);
            offsetVersionAndCreationTimeList = ImmutableList.of(new Triple<>(rawOffset, version, creationTime));
        } else {
            offsetVersionAndCreationTimeList = new ArrayList(ImmutableList.copyOf(offsetListStore.getOffsetList(-rawOffset).offsetVersionAndStorageTimeIterable()));
            Collections.sort(offsetVersionAndCreationTimeList, new OffsetVersionAndCreationTimeListReverseComparator());
        }
        // List is now in reverse order; iterate down through the offsets
        for (Triple<Integer, Long, Long> offsetVersionAndCreationTime : offsetVersionAndCreationTimeList) {
            DHTKey entryKey;
            int offset;
            long version;
            long creationTime;
            offset = offsetVersionAndCreationTime.getV1();
            version = offsetVersionAndCreationTime.getV2();
            creationTime = offsetVersionAndCreationTime.getV3();
            entryKey = entry.getKey();
            // Leaving for now as tracking in memory requires space
            if (vrp.retains(entryKey, version, creationTime, isInvalidated(offset), valueRetentionState, curTimeNanos) && ringMaster.iAmPotentialReplicaFor(entryKey)) {
                ++numRetained;
                retainedOffsets.add(offset);
            // Log.warningf("Retained %s\t%d", entry.getKey(), offset);
            } else {
                ++numDiscarded;
                discardedOffsets.add(offset);
            // Log.warningf("Discarded %s\t%d", entry.getKey(), offset);
            }
        }
    }
    return new Triple<>(new CompactionCheckResult(numRetained, numDiscarded), retainedOffsets, discardedOffsets);
}

Example 40 with DHTKey

use of com.ms.silverking.cloud.dht.common.DHTKey in project SilverKing by Morgan-Stanley.

the class DataSegmentWalker method next.

/**
 * Position at next entry. Initial call
 * @return if there is an additional entry
 */
@Override
public DataSegmentWalkEntry next() {
    long msl;
    long lsl;
    int storedLength;
    int uncompressedLength;
    int compressedLength;
    long nextMSL;
    long nextLSL;
    if (!hasNext) {
        return null;
    } else {
        DHTKey curKey;
        int curOffset;
        long version;
        long creationTime;
        // double  keyEntropy;
        byte storageState;
        ValueCreator creator;
        ByteBuffer entry;
        int nextEntryPostKeyPosition;
        if (debug) {
            System.out.println("position: " + position);
        }
        msl = dataSegment.getLong(position);
        lsl = dataSegment.getLong(position + NumConversion.BYTES_PER_LONG);
        curOffset = position;
        position += DHTKey.BYTES_PER_KEY;
        curKey = new SimpleKey(msl, lsl);
        // FUTURE - probably delete the below code, but could consider some
        // additional sanity checking
        /*
            keyEntropy = KeyUtil.keyEntropy(curKey);            
            if (keyEntropy < minValidKeyEntropy) {
                boolean sane;
                
                Log.warning("Invalid key: ", curKey +" "+ position);
                sane = false;
                // FUTURE - need more sophisticated validation of entry
                // FUTURE - we need to decode the entry
                while (!sane) {
                    if (position > dataSegment.limit() - DHTKey.BYTES_PER_KEY) { // FUTURE - crude limit; refine
                        Log.warning("Ran off the end of the segment searching for a valid key");
                        return null;
                    }
                    position++;
                    msl = dataSegment.getLong(position);
                    lsl = dataSegment.getLong(position + NumConversion.BYTES_PER_LONG);
                    curKey = new SimpleKey(msl, lsl);
                    keyEntropy = KeyUtil.keyEntropy(curKey);

                    if (keyEntropy >= minValidKeyEntropy) {
                        try {
                            storedLength = MetaDataUtil.getStoredLength(dataSegment, position);
                            if (storedLength > 0) {
                                uncompressedLength = MetaDataUtil.getUncompressedLength(dataSegment, position);
                                if (uncompressedLength >= 0) {
                                    compressedLength = MetaDataUtil.getCompressedLength(dataSegment, position);
                                    if (compressedLength >= 0) {
                                        if (position + storedLength < dataSegment.limit() 
                                                && uncompressedLength < compressedLength) {
                                            sane = true;
                                        }
                                    }
                                }
                            }
                        } catch (Exception e) {
                            // couldn't decode entry, move to next
                        }
                    }
                }
            }
            
            if (debug) {
                System.out.printf("%x:%x %3.2f\n", msl, lsl, keyEntropy);
            }
            */
        storedLength = MetaDataUtil.getStoredLength(dataSegment, position);
        uncompressedLength = MetaDataUtil.getUncompressedLength(dataSegment, position);
        compressedLength = MetaDataUtil.getCompressedLength(dataSegment, position);
        version = MetaDataUtil.getVersion(dataSegment, position);
        creationTime = MetaDataUtil.getCreationTime(dataSegment, position);
        storageState = MetaDataUtil.getStorageState(dataSegment, position);
        creator = MetaDataUtil.getCreator(dataSegment, position);
        if (debug) {
            System.out.println(storedLength);
        }
        nextEntryPostKeyPosition = position + storedLength + DHTKey.BYTES_PER_KEY;
        // Check to see if it's possible that there is another entry
        if (nextEntryPostKeyPosition + MetaDataUtil.getMinimumEntrySize() < dataSegment.limit()) {
            int nextEntryStoredLength;
            // Check to see if the potential next entry actually fits in this segment
            nextEntryStoredLength = MetaDataUtil.getStoredLength(dataSegment, nextEntryPostKeyPosition);
            if (debug) {
                System.out.printf("nextEntryPostKeyPosition %d nextEntryStoredLength %d dataSegment.limit() %d\n", nextEntryPostKeyPosition, nextEntryStoredLength, dataSegment.limit());
            }
            if (nextEntryPostKeyPosition + nextEntryStoredLength < dataSegment.limit()) {
                nextMSL = dataSegment.getLong(nextEntryPostKeyPosition);
                nextLSL = dataSegment.getLong(nextEntryPostKeyPosition + NumConversion.BYTES_PER_LONG);
                hasNext = !(nextMSL == 0 && nextLSL == 0);
                if (debug) {
                    System.out.printf("a: hastNext %s\n", hasNext);
                }
            } else {
                hasNext = false;
                if (debug) {
                    System.out.printf("b: hastNext %s\n", hasNext);
                }
            }
        } else {
            // No room for valid next entry
            hasNext = false;
            if (debug) {
                System.out.printf("c: hastNext %s\n", hasNext);
            }
        }
        entry = (ByteBuffer) ((ByteBuffer) dataSegment.duplicate().position(position)).slice().limit(storedLength);
        position += storedLength;
        return new DataSegmentWalkEntry(curKey, version, curOffset, storedLength, uncompressedLength, compressedLength, DHTKey.BYTES_PER_KEY, entry, creationTime, creator, storageState);
    }
}