Examples with SimpleStopwatch com.ms.silverking.time.SimpleStopwatch used on opensource projects

Example 16 with SimpleStopwatch

use of com.ms.silverking.time.SimpleStopwatch in project SilverKing by Morgan-Stanley.

the class SyncController method waitForCompletion.

public void waitForCompletion(long time, TimeUnit unit) throws ConvergenceException {
    Timer timer;
    Timer statusTimer;
    scanForEligibleActions();
    timer = new SimpleTimer(unit, time);
    statusTimer = new SimpleTimer(statusComputationIntervalUnit, statusComputationIntervalMillis);
    lock.lock();
    try {
        boolean nonZeroCompletions;
        Stopwatch resendSW;
        Stopwatch alertSW;
        boolean incompleteDisplayed;
        incompleteDisplayed = false;
        nonZeroCompletions = false;
        alertSW = new SimpleStopwatch();
        resendSW = new SimpleStopwatch();
        while (!complete() && !timer.hasExpired()) {
            if (!nonZeroCompletions) {
                ThreadUtil.sleep(1);
            }
            if (statusTimer.hasExpired()) {
                computeStatus();
                statusTimer.reset();
            }
            sendNonConflictingRequests();
            nonZeroCompletions = serviceCompletionQueue(timer.getRemainingMillisLong(), TimeUnit.MILLISECONDS);
            if (nonZeroCompletions) {
                incompleteDisplayed = false;
                alertSW.reset();
            } else {
                if (alertSW.getSplitSeconds() > nonCompletionAlertThresholdSeconds && !incompleteDisplayed) {
                    incompleteDisplayed = true;
                    displayIncomplete();
                }
            }
            if (resendSW.getSplitSeconds() > resendCheckIntervalSeconds) {
                checkForResends();
                resendSW.reset();
            }
        }
        if (hasErrors) {
            throw new ConvergenceException("SyncController unable to complete due to errors");
        } else if (abandoned) {
            throw new ConvergenceException("Abandoned");
        } else if (!complete()) {
            throw new ConvergenceException("Sync timed out");
        }
    } finally {
        lock.unlock();
    }
}

Example 17 with SimpleStopwatch

use of com.ms.silverking.time.SimpleStopwatch in project SilverKing by Morgan-Stanley.

the class NamespaceStore method singleReverseSegmentWalk.

public <T extends ValueRetentionState> void singleReverseSegmentWalk(ValueRetentionPolicy<T> vrp, T valueRetentionState, long curTimeNanos) {
    Set<Integer> deletedSegments;
    HashedSetMap<DHTKey, Triple<Long, Integer, Long>> removedEntries;
    removedEntries = new HashedSetMap<>();
    deletedSegments = new HashSet<>();
    for (int i = headSegment.getSegmentNumber() - 1; i >= 0; i--) {
        if (segmentExists(i)) {
            Triple<CompactionCheckResult, Set<Integer>, Set<Integer>> result;
            WritableSegmentBase segment;
            CompactionCheckResult ccr;
            try {
                Stopwatch sw;
                sw = new SimpleStopwatch();
                if (nsOptions.getStorageType() == StorageType.RAM) {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                } else {
                    segment = getSegment(i, SegmentPrereadMode.Preread);
                // segment = FileSegment.openReadOnly(nsDir, i, nsOptions.getSegmentSize(), nsOptions, SegmentIndexLocation.RAM, SegmentPrereadMode.Preread);
                }
                try {
                    result = segment.singleReverseSegmentWalk(vrp, valueRetentionState, curTimeNanos, ringMaster);
                    ccr = result.getV1();
                } finally {
                    if (nsOptions.getStorageType() == StorageType.FILE) {
                        if (segment.getSegmentNumber() != headSegment.getSegmentNumber()) {
                            ((FileSegment) segment).removeReference();
                        }
                    }
                }
                Log.warningAsyncf("Segment %3d CompactionCheckResult:\t%s", i, ccr.toString());
                if (segment.getSegmentNumber() == headSegment.getSegmentNumber()) {
                    Log.warningAsyncf("Retaining head segment");
                } else {
                    if (ccr.getValidEntries() == 0) {
                        try {
                            recentFileSegments.remove(i);
                            FileSegmentCompactor.delete(nsDir, i);
                            deletedSegments.add(i);
                            if (FileSegment.mapEverything) {
                                ((FileSegment) segment).close();
                            }
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "Failed to delete segment: " + i);
                        }
                    } else if (ccr.getInvalidFraction() >= compactionThreshold) {
                        try {
                            HashedSetMap<DHTKey, Triple<Long, Integer, Long>> segmentRemovedEntries;
                            recentFileSegments.remove(i);
                            segmentRemovedEntries = FileSegmentCompactor.compact(nsDir, i, nsOptions, new RetainedOffsetMapCheck(result.getV2(), result.getV3()));
                            removedEntries.addAll(segmentRemovedEntries);
                        } catch (IOException ioe) {
                            Log.logErrorWarning(ioe, "IOException compacting segment: " + i);
                        }
                    }
                }
                sw.stop();
                Log.warningAsyncf("\t\t%d %f", i, sw.getElapsedSeconds());
            } catch (Exception e) {
                Log.logErrorWarning(e, "Skipping segment " + i + " due to Exception");
            }
        }
    }
    updateOffsetLists(deletedSegments, removedEntries);
}

Example 18 with SimpleStopwatch

use of com.ms.silverking.time.SimpleStopwatch in project SilverKing by Morgan-Stanley.

the class FileSegmentRecoverer method readFullSegment.

void readFullSegment(int segmentNumber, SegmentIndexLocation segmentIndexLocation, SegmentPrereadMode segmentPrereadMode) {
    Stopwatch sw;
    Log.warning("Reading full segment: ", segmentNumber);
    sw = new SimpleStopwatch();
    try {
        FileSegment segment;
        NamespaceProperties nsProperties;
        NamespaceOptions nsOptions;
        // DataSegmentWalker       dsWalker;
        nsProperties = NamespacePropertiesIO.read(nsDir);
        nsOptions = nsProperties.getOptions();
        segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsOptions.getSegmentSize(), nsOptions, segmentIndexLocation, segmentPrereadMode);
        for (DHTKeyIntEntry entry : segment.getPKC()) {
            int offset;
            offset = entry.getValue();
            if (offset < 0) {
                OffsetList offsetList;
                offsetList = segment.offsetListStore.getOffsetList(-offset);
                for (int listOffset : offsetList) {
                    System.out.printf("%s\t%d\t%d\t%f\t*%d\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(listOffset), KeyUtil.keyEntropy(entry), offset, offsetList);
                // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(listOffset));
                }
            } else {
                System.out.printf("%s\t%d\t%d\t%f\t%d\n", entry, segment.getCreationTime(offset), segment.getVersion(offset), KeyUtil.keyEntropy(entry), offset);
            // nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, segment.getVersion(offset));
            }
        }
        // dsWalker = new DataSegmentWalker(segment.dataBuf);
        // for (DataSegmentWalkEntry entry : dsWalker) {
        // nsStore.addToSizeStats(entry.getUncompressedLength(), entry.getCompressedLength());
        // }
        segment.close();
        sw.stop();
        Log.warning("Done reading full segment: ", segmentNumber + " " + sw);
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe, "Unable to recover: " + segmentNumber);
    }
}

Example 19 with SimpleStopwatch

use of com.ms.silverking.time.SimpleStopwatch in project SilverKing by Morgan-Stanley.

the class FileSegmentRecoverer method recoverFullSegment.

FileSegment recoverFullSegment(int segmentNumber, NamespaceStore nsStore, SegmentIndexLocation segmentIndexLocation, SegmentPrereadMode segmentPrereadMode) {
    FileSegment _segment;
    Log.warningf("Recovering full segment: %d %s", segmentNumber, segmentPrereadMode);
    _segment = null;
    try {
        FileSegment segment;
        Stopwatch sw;
        sw = new SimpleStopwatch();
        segment = FileSegment.openReadOnly(nsDir, segmentNumber, nsStore.getNamespaceOptions().getSegmentSize(), nsStore.getNamespaceOptions(), segmentIndexLocation, segmentPrereadMode);
        for (DHTKeyIntEntry entry : segment.getPKC()) {
            int offset;
            long creationTime;
            offset = entry.getValue();
            if (offset < 0) {
                OffsetList offsetList;
                offsetList = segment.offsetListStore.getOffsetList(-offset);
                for (Triple<Integer, Long, Long> offsetVersionAndStorageTime : offsetList.offsetVersionAndStorageTimeIterable()) {
                    creationTime = offsetVersionAndStorageTime.getV3();
                    nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, offsetVersionAndStorageTime.getV2(), creationTime);
                }
            } else {
                long version;
                if (nsStore.getNamespaceOptions().getVersionMode() == NamespaceVersionMode.SINGLE_VERSION) {
                    version = DHTConstants.unspecifiedVersion;
                } else {
                    version = segment.getVersion(offset);
                }
                if (nsStore.getNamespaceOptions().getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS) {
                    creationTime = segment.getCreationTime(offset);
                } else {
                    creationTime = 0;
                }
                nsStore.putSegmentNumberAndVersion(entry.getKey(), segmentNumber, version, creationTime);
            }
        }
        if (segmentPrereadMode != SegmentPrereadMode.Preread) {
            segment.close();
        } else {
            _segment = segment;
        }
        sw.stop();
        Log.warning("Done recovering full segment: ", segmentNumber + "\t" + sw.getElapsedSeconds());
    } catch (IOException ioe) {
        Log.logErrorWarning(ioe, "Unable to recover: " + segmentNumber);
    }
    return _segment;
}

Example 20 with SimpleStopwatch

use of com.ms.silverking.time.SimpleStopwatch in project SilverKing by Morgan-Stanley.

the class DevTest method test.

public void test(int numKeys, int reps, String namespace, Compression compression, ChecksumType checksumType, EnumSet<Test> tests) throws Exception {
    DHTClient client;
    ClientDHTConfiguration dhtConfig;
    DHTSession session;
    AsynchronousNamespacePerspective<String, String> asyncNSP;
    AsyncRetrieval<String, String> asyncRetrieval;
    AsyncPut<String> asyncPut;
    Stopwatch sw;
    PutOptions putOptions;
    asyncPut = null;
    client = new DHTClient();
    dhtConfig = new ClientDHTConfiguration(dhtName, new ZooKeeperConfig(zkLocs));
    session = client.openSession(dhtConfig);
    putOptions = session.getDefaultNamespaceOptions().getDefaultPutOptions().compression(compression).checksumType(checksumType);
    asyncNSP = session.openAsyncNamespacePerspective(namespace, String.class, String.class);
    sw = new SimpleStopwatch();
    if (tests.contains(Test.Put)) {
        System.out.println("\n\n\t\tPUT");
        for (int i = 0; i < reps; i++) {
            // asyncPut = asyncNSP.put("Hello"+ i, "world!");
            asyncPut = asyncNSP.put(createMap(i, numKeys), putOptions);
            asyncPut.waitForCompletion();
        }
        sw.stop();
        displayTimes(sw, reps, numKeys, valueSize);
    }
    if (tests.contains(Test.Get)) {
        System.out.println("\n\n\t\tGET");
        GetOptions getOptions;
        sw.reset();
        getOptions = OptionsHelper.newGetOptions(RetrievalType.VALUE_AND_META_DATA, session.getDefaultNamespaceOptions().getDefaultGetOptions().getVersionConstraint());
        for (int i = 0; i < reps; i++) {
            Set<String> keys;
            Map<String, ? extends StoredValue<String>> values;
            keys = createSet(i, numKeys);
            asyncRetrieval = asyncNSP.get(keys, getOptions);
            asyncRetrieval.waitForCompletion();
            if (displayValues) {
                System.out.printf("keys: %s\n", CollectionUtil.toString(keys));
                values = asyncRetrieval.getStoredValues();
                System.out.printf("values: %s\n", CollectionUtil.toString(values.entrySet()));
                for (Entry<String, ? extends StoredValue<String>> entry : values.entrySet()) {
                    System.out.println(entry.getKey() + " -> " + entry.getValue().getValue() + "\t" + entry.getValue().getMetaData().toString(true));
                }
            }
        }
        sw.stop();
        displayTimes(sw, reps, numKeys, valueSize);
    }
}