Examples with HashedSetMap com.ms.silverking.collection.HashedSetMap used on opensource projects

Example 1 with HashedSetMap

use of com.ms.silverking.collection.HashedSetMap in project SilverKing by Morgan-Stanley.

the class NamespaceStore method singleReverseSegmentWalk.

public <T extends ValueRetentionState> void singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos) {
    Set<Integer> deletedSegments;
    HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries;
    removedEntries = new HashedSetMap<>();
    deletedSegments = new HashSet<>();
    for (int i = headSegment.getSegmentNumber() - 1; i >= 0; i--) {
        if (segmentExists(i)) {
            Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> result;
            WritableSegmentBase segment;
            CompactionCheckResult ccr;
            try {
                Stopwatch sw;
                sw = new SimpleStopwatch();
                if (nsOptions.getStorageType() == StorageType.RAM) {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                } else {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                // segment = FileSegment.openReadOnly(nsDir, i, nsOptions.getSegmentSize(), nsOptions, SegmentIndexLocation.RAM, SegmentPrereadMode.Preread);
                }
                try {
                    result = segment.singleReverseSegmentWalk(vrp, valueRetentionState, curTimeNanos, ringMaster);
                    ccr = result.getV1();
                } finally {
                    if (nsOptions.getStorageType() == StorageType.FILE) {
                        if (segment.getSegmentNumber() != headSegment.getSegmentNumber()) {
                            ((FileSegment) segment).removeReference();
                        }
                    }
                }
                Log.warningAsyncf("Segment %3d CompactionCheckResult:\t%s", i, ccr.toString());
                if (segment.getSegmentNumber() == headSegment.getSegmentNumber()) {
                    Log.warningAsyncf("Retaining head segment");
                } else {
                    if (ccr.getValidEntries() == 0) {
                        try {
                            recentFileSegments.remove(i);
                            FileSegmentCompactor.delete(nsDir, i);
                            deletedSegments.add(i);
                            if (FileSegment.mapEverything) {
                                ((FileSegment) segment).close();
                            }
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "Failed to delete segment: " + i);
                        }
                    } else if (ccr.getInvalidFraction() >= compactionThreshold) {
                        try {
                            HashedSetMap<DHTKey, Triple<Long, Integer, Long>> segmentRemovedEntries;
                            recentFileSegments.remove(i);
                            segmentRemovedEntries = FileSegmentCompactor.compact(nsDir, i, nsOptions, new RetainedOffsetMapCheck(result.getV2(), result.getV3()));
                            removedEntries.addAll(segmentRemovedEntries);
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "IOException compacting segment: " + i);
                        }
                    }
                }
                sw.stop();
                Log.warningAsyncf("\t\t%d %f", i, sw.getElapsedSeconds());
            } catch (Exception e) {
                Log.logErrorWarning(e, "Skipping segment " + i + " due to Exception");
            }
        }
    }
    updateOffsetLists(deletedSegments, removedEntries);
}