Examples with DHTKeyIntEntry com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry used on opensource projects

Example 1 with DHTKeyIntEntry

use of com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry in project SilverKing by Morgan-Stanley.

the class FileSegmentRecoverer method readFullSegment.

void readFullSegment(int segmentNumber, SegmentIndexLocation segmentIndexLocation, SegmentPrereadMode segmentPrereadMode) {
    Stopwatch sw;
    Log.warning("Reading full segment: ", segmentNumber);
    sw = new SimpleStopwatch();
    try {
        FileSegment segment;
        NamespaceProperties nsProperties;
        NamespaceOptions nsOptions;
        // DataSegmentWalker       dsWalker;
        nsProperties = NamespacePropertiesIO.read(nsDir);
        nsOptions = nsProperties.getOptions();
        segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsOptions.getSegmentSize(), nsOptions, segmentIndexLocation, segmentPrereadMode);
        for (DHTKeyIntEntry entry : segment.getPKC()) {
            int offset;
            offset = entry.getValue();
            if (offset < 0) {
                OffsetList offsetList;
                offsetList = segment.offsetListStore.getOffsetList(-offset);
                for (int listOffset : offsetList) {
                    System.out.printf("%s\t%d\t%d\t%f\t*%d\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(listOffset), KeyUtil.keyEntropy(entry), offset, offsetList);
                // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(listOffset));
                }
            } else {
                System.out.printf("%s\t%d\t%d\t%f\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(offset), KeyUtil.keyEntropy(entry), offset);
            // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(offset));
            }
        }
        // dsWalker = new DataSegmentWalker(segment.dataBuf);
        // for (DataSegmentWalkEntry entry : dsWalker) {
        // nsStore.addToSizeStats(entry.getUncompressedLength(), entry.getCompressedLength());
        // }
        segment.close();
        sw.stop();
        Log.warning("Done reading full segment: ", segmentNumber + " " + sw);
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe, "Unable to recover: " + segmentNumber);
    }
}

Example 2 with DHTKeyIntEntry

use of com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry in project SilverKing by Morgan-Stanley.

the class FileSegmentRecoverer method recoverFullSegment.

FileSegment recoverFullSegment(int segmentNumber, NamespaceStore nsStore, SegmentIndexLocation segmentIndexLocation, SegmentPrereadMode segmentPrereadMode) {
    FileSegment _segment;
    Log.warningf("Recovering full segment: %d %s", segmentNumber, segmentPrereadMode);
    _segment = null;
    try {
        FileSegment segment;
        Stopwatch sw;
        sw = new SimpleStopwatch();
        segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsStore.getNamespaceOptions().getSegmentSize(), nsStore.getNamespaceOptions(), segmentIndexLocation, segmentPrereadMode);
        for (DHTKeyIntEntry entry : segment.getPKC()) {
            int offset;
            long creationTime;
            offset = entry.getValue();
            if (offset < 0) {
                OffsetList offsetList;
                offsetList = segment.offsetListStore.getOffsetList(-offset);
                for (Triple<Integer, Long, Long> offsetVersionAndStorageTime : offsetList.offsetVersionAndStorageTimeIterable()) {
                    creationTime = offsetVersionAndStorageTime.getV3();
                    nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, offsetVersionAndStorageTime.getV2(), creationTime);
                }
            } else {
                long version;
                if (nsStore.getNamespaceOptions().getVersionMode() == NamespaceVersionMode.SINGLE_VERSION) {
                    version = DHTConstants.unspecifiedVersion;
                } else {
                    version = segment.getVersion(offset);
                }
                if (nsStore.getNamespaceOptions().getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS) {
                    creationTime = segment.getCreationTime(offset);
                } else {
                    creationTime = 0;
                }
                nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, version, creationTime);
            }
        }
        if (segmentPrereadMode != SegmentPrereadMode.Preread) {
            segment.close();
        } else {
            _segment = segment;
        }
        sw.stop();
        Log.warning("Done recovering full segment: ", segmentNumber + "\t" + sw.getElapsedSeconds());
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe, "Unable to recover: " + segmentNumber);
    }
    return _segment;
}

Example 3 with DHTKeyIntEntry

use of com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry in project SilverKing by Morgan-Stanley.

the class NamespaceStore method updateOffsetLists.

private void updateOffsetLists(Set<Integer> deletedSegments, HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries) {
    RAMOffsetListStore ols;
    ols = (RAMOffsetListStore) offsetListStore;
    if (deletedSegments.size() > 0) {
        Set<DHTKey> singleKeysInDeletedSegments;
        for (int i = 1; i <= ols.getNumLists(); i++) {
            // offset list indexing is 1-based
            RAMOffsetList ol;
            ol = (RAMOffsetList) ols.getOffsetList(i);
            ol.removeEntriesByValue(deletedSegments);
        }
        singleKeysInDeletedSegments = new HashSet<>();
        for (DHTKeyIntEntry keyAndSegment : valueSegments) {
            if (keyAndSegment.getValue() >= 0 && deletedSegments.contains(keyAndSegment.getValue())) {
                singleKeysInDeletedSegments.add(keyAndSegment.getKey());
            }
        }
        for (DHTKey singleKeyToRemove : singleKeysInDeletedSegments) {
            valueSegments.remove(singleKeyToRemove);
        }
    }
    for (DHTKey key : removedEntries.getKeys()) {
        int rawSegmentNumber;
        rawSegmentNumber = valueSegments.get(key);
        if (rawSegmentNumber < 0) {
            if (rawSegmentNumber != IntCuckooConstants.noSuchValue) {
                RAMOffsetList ol;
                ol = (RAMOffsetList) ols.getOffsetList(-rawSegmentNumber);
                ol.removeEntriesByMatch(removedEntries.getSet(key));
            } else {
            // No action required
            }
        } else {
            long[] versionAndStorageTime;
            long creationTime;
            // only one should exist
            versionAndStorageTime = segmentOldestVersion(rawSegmentNumber, key);
            if (removedEntries.getSet(key).size() != 1) {
                Log.warningAsyncf("Unexpected removedEntries.getSet(key).size() != 1");
            }
            creationTime = nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS ? versionAndStorageTime[1] : 0;
            if (removedEntries.getSet(key).contains(new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime))) {
                valueSegments.remove(key);
            } else {
                Log.warningAsyncf("Unexpected !removedEntries.getSet(key).contains(new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime))");
                Log.warningAsyncf("%s", new Triple<>(versionAndStorageTime[0], rawSegmentNumber, creationTime));
            }
        }
    }
}

Example 4 with DHTKeyIntEntry

use of com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry in project SilverKing by Morgan-Stanley.

the class WritableSegmentBase method singleReverseSegmentWalk.

public <T extends ValueRetentionState> Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos, NodeRingMaster2 ringMaster) {
    int numRetained;
    int numDiscarded;
    Set<Integer> retainedOffsets;
    Set<Integer> discardedOffsets;
    numRetained = 0;
    numDiscarded = 0;
    retainedOffsets = new HashSet<>();
    discardedOffsets = new HashSet<>();
    // The outer loop loops through the keys first.
    for (DHTKeyIntEntry entry : keyToOffset) {
        int rawOffset;
        List<Triple<Integer, Long, Long>> offsetVersionAndCreationTimeList;
        rawOffset = entry.getValue();
        if (rawOffset >= 0) {
            long version;
            long creationTime;
            version = getVersion(rawOffset);
            creationTime = getCreationTime(rawOffset);
            offsetVersionAndCreationTimeList = ImmutableList.of(new Triple<>(rawOffset, version, creationTime));
        } else {
            offsetVersionAndCreationTimeList = new ArrayList(ImmutableList.copyOf(offsetListStore.getOffsetList(-rawOffset).offsetVersionAndStorageTimeIterable()));
            Collections.sort(offsetVersionAndCreationTimeList, new OffsetVersionAndCreationTimeListReverseComparator());
        }
        // List is now in reverse order; iterate down through the offsets
        for (Triple<Integer, Long, Long> offsetVersionAndCreationTime : offsetVersionAndCreationTimeList) {
            DHTKey entryKey;
            int offset;
            long version;
            long creationTime;
            offset = offsetVersionAndCreationTime.getV1();
            version = offsetVersionAndCreationTime.getV2();
            creationTime = offsetVersionAndCreationTime.getV3();
            entryKey = entry.getKey();
            // Leaving for now as tracking in memory requires space
            if (vrp.retains(entryKey, version, creationTime, isInvalidated(offset), valueRetentionState, curTimeNanos) && ringMaster.iAmPotentialReplicaFor(entryKey)) {
                ++numRetained;
                retainedOffsets.add(offset);
            // Log.warningf("Retained %s\t%d", entry.getKey(), offset);
            } else {
                ++numDiscarded;
                discardedOffsets.add(offset);
            // Log.warningf("Discarded %s\t%d", entry.getKey(), offset);
            }
        }
    }
    return new Triple<>(new CompactionCheckResult(numRetained, numDiscarded), retainedOffsets, discardedOffsets);
}

Example 5 with DHTKeyIntEntry

use of com.ms.silverking.cloud.dht.collection.DHTKeyIntEntry in project SilverKing by Morgan-Stanley.

the class FileSegmentUtil method debugSegment.

public void debugSegment(DHTKey key, File nsDir, int segmentNumber, NamespaceOptions nsOptions) throws IOException {
    FileSegment segment;
    System.out.printf("Segment %d\n", segmentNumber);
    segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsOptions.getSegmentSize(), nsOptions);
    for (DHTKeyIntEntry entry : segment.getPKC()) {
        // System.out.printf("%s\t%s\t%s\n", KeyUtil.keyToString(entry.getKey()), KeyUtil.keyToString(key), entry.getKey().equals(key));
        if (key == null || entry.getKey().equals(key)) {
            int offset;
            offset = entry.getValue();
            if (key == null) {
                if (offset < 0) {
                    OffsetList offsetList;
                    offsetList = segment.offsetListStore.getOffsetList(-offset);
                    for (int listOffset : offsetList) {
                        System.out.printf("%s\t%d\n", entry.getKey(), listOffset);
                    // displayValue(segment.retrieveForDebug(key, listOffset));
                    }
                } else {
                    System.out.printf("%s\t%d\n", entry.getKey(), offset);
                // displayValue(segment.retrieveForDebug(key, offset));
                }
            } else {
                System.out.printf("%s\t%d\n", KeyUtil.keyToString(key), offset);
                if (offset < 0) {
                    OffsetList offsetList;
                    offsetList = segment.offsetListStore.getOffsetList(-offset);
                    for (int listOffset : offsetList) {
                        displayValue(segment.retrieveForDebug(key, listOffset));
                    }
                } else {
                    displayValue(segment.retrieveForDebug(key, offset));
                }
            }
        }
    }
}