Examples with Triple com.ms.silverking.collection.Triple used on opensource projects

Example 1 with Triple

use of com.ms.silverking.collection.Triple in project SilverKing by Morgan-Stanley.

the class DF method df.

public static Triple<Long, Long, Integer> df(String path, int timeoutInSeconds) throws IOException, RuntimeException {
    ProcessExecutor pe;
    String[] commands;
    String rawOutput;
    commands = new String[3];
    commands[0] = dfCommand;
    commands[1] = "-k";
    commands[2] = path;
    pe = new ProcessExecutor(commands, timeoutInSeconds);
    pe.execute();
    if (pe.getExitCode() != 0) {
        throw new RuntimeException("non-zero df exit code: " + pe.getExitCode());
    }
    rawOutput = pe.getOutput();
    if (rawOutput == null) {
        throw new RuntimeException("null df output: ");
    } else {
        String[] output;
        int line2Index;
        line2Index = rawOutput.indexOf('/');
        if (line2Index < 0) {
            throw new RuntimeException("Unexpected df output: " + rawOutput);
        } else {
            String line2;
            line2 = rawOutput.substring(line2Index);
            output = line2.split("\\s+");
            try {
                long totalBlocks;
                long usedBlocks;
                totalBlocks = Long.parseLong(output[1]);
                usedBlocks = Long.parseLong(output[2]);
                return new Triple<>(totalBlocks, usedBlocks, blockSize);
            } catch (RuntimeException re) {
                Log.logErrorWarning(re);
                throw new RuntimeException("Exception parsing output: " + rawOutput, re);
            }
        }
    }
}

Example 2 with Triple

use of com.ms.silverking.collection.Triple in project SilverKing by Morgan-Stanley.

the class NamespaceStore method singleReverseSegmentWalk.

public <T extends ValueRetentionState> void singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos) {
    Set<Integer> deletedSegments;
    HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries;
    removedEntries = new HashedSetMap<>();
    deletedSegments = new HashSet<>();
    for (int i = headSegment.getSegmentNumber() - 1; i >= 0; i--) {
        if (segmentExists(i)) {
            Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> result;
            WritableSegmentBase segment;
            CompactionCheckResult ccr;
            try {
                Stopwatch sw;
                sw = new SimpleStopwatch();
                if (nsOptions.getStorageType() == StorageType.RAM) {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                } else {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                // segment = FileSegment.openReadOnly(nsDir, i, nsOptions.getSegmentSize(), nsOptions, SegmentIndexLocation.RAM, SegmentPrereadMode.Preread);
                }
                try {
                    result = segment.singleReverseSegmentWalk(vrp, valueRetentionState, curTimeNanos, ringMaster);
                    ccr = result.getV1();
                } finally {
                    if (nsOptions.getStorageType() == StorageType.FILE) {
                        if (segment.getSegmentNumber() != headSegment.getSegmentNumber()) {
                            ((FileSegment) segment).removeReference();
                        }
                    }
                }
                Log.warningAsyncf("Segment %3d CompactionCheckResult:\t%s", i, ccr.toString());
                if (segment.getSegmentNumber() == headSegment.getSegmentNumber()) {
                    Log.warningAsyncf("Retaining head segment");
                } else {
                    if (ccr.getValidEntries() == 0) {
                        try {
                            recentFileSegments.remove(i);
                            FileSegmentCompactor.delete(nsDir, i);
                            deletedSegments.add(i);
                            if (FileSegment.mapEverything) {
                                ((FileSegment) segment).close();
                            }
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "Failed to delete segment: " + i);
                        }
                    } else if (ccr.getInvalidFraction() >= compactionThreshold) {
                        try {
                            HashedSetMap<DHTKey, Triple<Long, Integer, Long>> segmentRemovedEntries;
                            recentFileSegments.remove(i);
                            segmentRemovedEntries = FileSegmentCompactor.compact(nsDir, i, nsOptions, new RetainedOffsetMapCheck(result.getV2(), result.getV3()));
                            removedEntries.addAll(segmentRemovedEntries);
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "IOException compacting segment: " + i);
                        }
                    }
                }
                sw.stop();
                Log.warningAsyncf("\t\t%d %f", i, sw.getElapsedSeconds());
            } catch (Exception e) {
                Log.logErrorWarning(e, "Skipping segment " + i + " due to Exception");
            }
        }
    }
    updateOffsetLists(deletedSegments, removedEntries);
}

Example 3 with Triple

use of com.ms.silverking.collection.Triple in project SilverKing by Morgan-Stanley.

the class NamespaceStore method _retrieve.

protected ByteBuffer[] _retrieve(DHTKey[] keys, InternalRetrievalOptions options) {
    int[] segmentNumbers;
    Triple<DHTKey, Integer, Integer>[] keysSegmentNumbersAndIndices;
    ByteBuffer[] sortedResults;
    ByteBuffer[] unsortedResults;
    segmentNumbers = new int[keys.length];
    for (int i = 0; i < keys.length; i++) {
        segmentNumbers[i] = getSegmentNumber(keys[i], options.getVersionConstraint());
        if (debugParent) {
            Log.warningf("_retrieve %x %s segmentNumber %d", ns, keys[i], segmentNumbers[i]);
        }
    }
    // Sort so that we group segment access. Not always useful,
    // but can become critical for large retrievals that span multiple segments
    keysSegmentNumbersAndIndices = new Triple[keys.length];
    for (int i = 0; i < keysSegmentNumbersAndIndices.length; i++) {
        keysSegmentNumbersAndIndices[i] = new Triple<>(keys[i], segmentNumbers[i], i);
    }
    Arrays.sort(keysSegmentNumbersAndIndices, KeyAndSegmentComparator.mostRecentSegmentsFirst);
    sortedResults = _retrieve(keysSegmentNumbersAndIndices, options);
    // We fix up the ordering here. It would be possible to avoid this
    // by constructing the unsorted results inside of the above _retrieve() call.
    // We currently avoid that to avoid leaking the notion of sorting/indexing changes
    // into that method.
    unsortedResults = new ByteBuffer[sortedResults.length];
    for (int i = 0; i < sortedResults.length; i++) {
        unsortedResults[keysSegmentNumbersAndIndices[i].getV3()] = sortedResults[i];
    }
    return unsortedResults;
}

Example 4 with Triple

use of com.ms.silverking.collection.Triple in project SilverKing by Morgan-Stanley.

the class WritableSegmentBase method singleReverseSegmentWalk.

public <T extends ValueRetentionState> Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos, NodeRingMaster2 ringMaster) {
    int numRetained;
    int numDiscarded;
    Set<Integer> retainedOffsets;
    Set<Integer> discardedOffsets;
    numRetained = 0;
    numDiscarded = 0;
    retainedOffsets = new HashSet<>();
    discardedOffsets = new HashSet<>();
    // The outer loop loops through the keys first.
    for (DHTKeyIntEntry entry : keyToOffset) {
        int rawOffset;
        List<Triple<Integer, Long, Long>> offsetVersionAndCreationTimeList;
        rawOffset = entry.getValue();
        if (rawOffset >= 0) {
            long version;
            long creationTime;
            version = getVersion(rawOffset);
            creationTime = getCreationTime(rawOffset);
            offsetVersionAndCreationTimeList = ImmutableList.of(new Triple<>(rawOffset, version, creationTime));
        } else {
            offsetVersionAndCreationTimeList = new ArrayList(ImmutableList.copyOf(offsetListStore.getOffsetList(-rawOffset).offsetVersionAndStorageTimeIterable()));
            Collections.sort(offsetVersionAndCreationTimeList, new OffsetVersionAndCreationTimeListReverseComparator());
        }
        // List is now in reverse order; iterate down through the offsets
        for (Triple<Integer, Long, Long> offsetVersionAndCreationTime : offsetVersionAndCreationTimeList) {
            DHTKey entryKey;
            int offset;
            long version;
            long creationTime;
            offset = offsetVersionAndCreationTime.getV1();
            version = offsetVersionAndCreationTime.getV2();
            creationTime = offsetVersionAndCreationTime.getV3();
            entryKey = entry.getKey();
            // Leaving for now as tracking in memory requires space
            if (vrp.retains(entryKey, version, creationTime, isInvalidated(offset), valueRetentionState, curTimeNanos) && ringMaster.iAmPotentialReplicaFor(entryKey)) {
                ++numRetained;
                retainedOffsets.add(offset);
            // Log.warningf("Retained %s\t%d", entry.getKey(), offset);
            } else {
                ++numDiscarded;
                discardedOffsets.add(offset);
            // Log.warningf("Discarded %s\t%d", entry.getKey(), offset);
            }
        }
    }
    return new Triple<>(new CompactionCheckResult(numRetained, numDiscarded), retainedOffsets, discardedOffsets);
}

Example 5 with Triple

use of com.ms.silverking.collection.Triple in project SilverKing by Morgan-Stanley.

the class FileSegmentCompactor method compact.

public static HashedSetMap<DHTKey, Triple<Long, Integer, Long>> compact(File nsDir, int segmentNumber, NamespaceOptions nsOptions, EntryRetentionCheck retentionCheck) throws IOException {
    FileSegment compactedSegment;
    File oldFile;
    File trashFile;
    File newFile;
    HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries;
    removedEntries = new HashedSetMap<>();
    compactedSegment = createCompactedSegment(nsDir, segmentNumber, nsOptions, nsOptions.getSegmentSize(), retentionCheck, removedEntries, nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS);
    compactedSegment.persist();
    Log.warning("Swapping to compacted segment: ", segmentNumber);
    oldFile = FileSegment.fileForSegment(nsDir, segmentNumber);
    newFile = getCompactionFile(nsDir, segmentNumber);
    trashFile = getTrashFile(nsDir, segmentNumber);
    // Leave old file around for one cycle in case there are references to it
    rename(oldFile, trashFile);
    rename(newFile, oldFile);
    Log.warning("Done swapping to compacted segment: ", segmentNumber);
    return removedEntries;
}