Examples with Stopwatch com.google.common.base.Stopwatch used on opensource projects

Example 41 with Stopwatch

use of com.google.common.base.Stopwatch in project druid by druid-io.

the class RealtimePlumber method persist.

@Override
public void persist(final Committer committer) {
    final List<Pair<FireHydrant, Interval>> indexesToPersist = Lists.newArrayList();
    for (Sink sink : sinks.values()) {
        if (sink.swappable()) {
            indexesToPersist.add(Pair.of(sink.swap(), sink.getInterval()));
        }
    }
    log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource());
    final Stopwatch runExecStopwatch = Stopwatch.createStarted();
    final Stopwatch persistStopwatch = Stopwatch.createStarted();
    final Map<String, Object> metadataElems = committer.getMetadata() == null ? null : ImmutableMap.of(COMMIT_METADATA_KEY, committer.getMetadata(), COMMIT_METADATA_TIMESTAMP_KEY, System.currentTimeMillis());
    persistExecutor.execute(new ThreadRenamingRunnable(String.format("%s-incremental-persist", schema.getDataSource())) {

        @Override
        public void doRun() {
            /* Note:
            If plumber crashes after storing a subset of all the hydrants then we will lose data and next
            time we will start with the commitMetadata stored in those hydrants.
            option#1:
            maybe it makes sense to store the metadata outside the segments in a separate file. This is because the
            commit metadata isn't really associated with an individual segment-- it's associated with a set of segments
            that are persisted at the same time or maybe whole datasource. So storing it in the segments is asking for problems.
            Sort of like this:

            {
              "metadata" : {"foo": "bar"},
              "segments": [
                {"id": "datasource_2000_2001_2000_1", "hydrant": 10},
                {"id": "datasource_2001_2002_2001_1", "hydrant": 12},
              ]
            }
            When a realtime node crashes and starts back up, it would delete any hydrants numbered higher than the
            ones in the commit file.

            option#2
            We could also just include the set of segments for the same chunk of metadata in more metadata on each
            of the segments. we might also have to think about the hand-off in terms of the full set of segments being
            handed off instead of individual segments being handed off (that is, if one of the set succeeds in handing
            off and the others fail, the real-time would believe that it needs to re-ingest the data).
             */
            long persistThreadCpuTime = VMUtils.safeGetThreadCpuTime();
            try {
                for (Pair<FireHydrant, Interval> pair : indexesToPersist) {
                    metrics.incrementRowOutputCount(persistHydrant(pair.lhs, schema, pair.rhs, metadataElems));
                }
                committer.run();
            } catch (Exception e) {
                metrics.incrementFailedPersists();
                throw e;
            } finally {
                metrics.incrementPersistCpuTime(VMUtils.safeGetThreadCpuTime() - persistThreadCpuTime);
                metrics.incrementNumPersists();
                metrics.incrementPersistTimeMillis(persistStopwatch.elapsed(TimeUnit.MILLISECONDS));
                persistStopwatch.stop();
            }
        }
    });
    final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS);
    metrics.incrementPersistBackPressureMillis(startDelay);
    if (startDelay > WARN_DELAY) {
        log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay);
    }
    runExecStopwatch.stop();
    resetNextFlush();
}

Example 42 with Stopwatch

use of com.google.common.base.Stopwatch in project graylog2-server by Graylog2.

the class InputSetupService method shutDown.

@Override
protected void shutDown() throws Exception {
    LOG.debug("Stopping InputSetupService");
    eventBus.unregister(this);
    for (IOState<MessageInput> state : inputRegistry.getRunningInputs()) {
        MessageInput input = state.getStoppable();
        LOG.info("Attempting to close input <{}> [{}].", input.getUniqueReadableId(), input.getName());
        Stopwatch s = Stopwatch.createStarted();
        try {
            input.stop();
            LOG.info("Input <{}> closed. Took [{}ms]", input.getUniqueReadableId(), s.elapsed(TimeUnit.MILLISECONDS));
        } catch (Exception e) {
            LOG.error("Unable to stop input <{}> [{}]: " + e.getMessage(), input.getUniqueReadableId(), input.getName());
        } finally {
            s.stop();
        }
    }
    LOG.debug("Stopped InputSetupService");
}

Example 43 with Stopwatch

use of com.google.common.base.Stopwatch in project graylog2-server by Graylog2.

the class MongoIndexRangeService method calculateRange.

@Override
public IndexRange calculateRange(String index) {
    indices.waitForRecovery(index);
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final Stopwatch sw = Stopwatch.createStarted();
    final IndexRangeStats stats = indices.indexRangeStatsOfIndex(index);
    final int duration = Ints.saturatedCast(sw.stop().elapsed(TimeUnit.MILLISECONDS));
    LOG.info("Calculated range of [{}] in [{}ms].", index, duration);
    return MongoIndexRange.create(index, stats.min(), stats.max(), now, duration, stats.streamIds());
}

Example 44 with Stopwatch

use of com.google.common.base.Stopwatch in project distributedlog by twitter.

the class BKLogSegmentWriter method addComplete.

@Override
public void addComplete(final int rc, LedgerHandle handle, final long entryId, final Object ctx) {
    final AtomicReference<Integer> effectiveRC = new AtomicReference<Integer>(rc);
    try {
        if (FailpointUtils.checkFailPoint(FailpointUtils.FailPointName.FP_TransmitComplete)) {
            effectiveRC.set(BKException.Code.UnexpectedConditionException);
        }
    } catch (Exception exc) {
        effectiveRC.set(BKException.Code.UnexpectedConditionException);
    }
    // Sanity check to make sure we're receiving these callbacks in order.
    if (entryId > -1 && lastEntryId >= entryId) {
        LOG.error("Log segment {} saw out of order entry {} lastEntryId {}", new Object[] { fullyQualifiedLogSegment, entryId, lastEntryId });
    }
    lastEntryId = entryId;
    assert (ctx instanceof BKTransmitPacket);
    final BKTransmitPacket transmitPacket = (BKTransmitPacket) ctx;
    // Time from transmit until receipt of addComplete callback
    addCompleteTime.registerSuccessfulEvent(TimeUnit.MICROSECONDS.convert(System.nanoTime() - transmitPacket.getTransmitTime(), TimeUnit.NANOSECONDS));
    if (BKException.Code.OK == rc) {
        EntryBuffer recordSet = transmitPacket.getRecordSet();
        if (recordSet.hasUserRecords()) {
            synchronized (this) {
                lastTxIdAcknowledged = Math.max(lastTxIdAcknowledged, recordSet.getMaxTxId());
            }
        }
    }
    if (null != addCompleteFuturePool) {
        final Stopwatch queuedTime = Stopwatch.createStarted();
        addCompleteFuturePool.apply(new Function0<Void>() {

            public Void apply() {
                final Stopwatch deferredTime = Stopwatch.createStarted();
                addCompleteQueuedTime.registerSuccessfulEvent(queuedTime.elapsed(TimeUnit.MICROSECONDS));
                addCompleteDeferredProcessing(transmitPacket, entryId, effectiveRC.get());
                addCompleteDeferredTime.registerSuccessfulEvent(deferredTime.elapsed(TimeUnit.MICROSECONDS));
                return null;
            }

            @Override
            public String toString() {
                return String.format("AddComplete(Stream=%s, entryId=%d, rc=%d)", fullyQualifiedLogSegment, entryId, rc);
            }
        }).addEventListener(new FutureEventListener<Void>() {

            @Override
            public void onSuccess(Void done) {
            }

            @Override
            public void onFailure(Throwable cause) {
                LOG.error("addComplete processing failed for {} entry {} lastTxId {} rc {} with error", new Object[] { fullyQualifiedLogSegment, entryId, transmitPacket.getRecordSet().getMaxTxId(), rc, cause });
            }
        });
        // Race condition if we notify before the addComplete is enqueued.
        transmitPacket.notifyTransmitComplete(effectiveRC.get());
        outstandingTransmits.getAndDecrement();
    } else {
        // Notify transmit complete must be called before deferred processing in the
        // sync case since otherwise callbacks in deferred processing may deadlock.
        transmitPacket.notifyTransmitComplete(effectiveRC.get());
        outstandingTransmits.getAndDecrement();
        addCompleteDeferredProcessing(transmitPacket, entryId, effectiveRC.get());
    }
}

Example 45 with Stopwatch

use of com.google.common.base.Stopwatch in project distributedlog by twitter.

the class BKLogWriteHandler method deleteLogSegment.

private Future<LogSegmentMetadata> deleteLogSegment(final LogSegmentMetadata ledgerMetadata) {
    LOG.info("Deleting ledger {} for {}", ledgerMetadata, getFullyQualifiedName());
    final Promise<LogSegmentMetadata> promise = new Promise<LogSegmentMetadata>();
    final Stopwatch stopwatch = Stopwatch.createStarted();
    promise.addEventListener(new FutureEventListener<LogSegmentMetadata>() {

        @Override
        public void onSuccess(LogSegmentMetadata segment) {
            deleteOpStats.registerSuccessfulEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        }

        @Override
        public void onFailure(Throwable cause) {
            deleteOpStats.registerFailedEvent(stopwatch.stop().elapsed(TimeUnit.MICROSECONDS));
        }
    });
    try {
        bookKeeperClient.get().asyncDeleteLedger(ledgerMetadata.getLedgerId(), new AsyncCallback.DeleteCallback() {

            @Override
            public void deleteComplete(int rc, Object ctx) {
                if (BKException.Code.NoSuchLedgerExistsException == rc) {
                    LOG.warn("No ledger {} found to delete for {} : {}.", new Object[] { ledgerMetadata.getLedgerId(), getFullyQualifiedName(), ledgerMetadata });
                } else if (BKException.Code.OK != rc) {
                    BKException bke = BKException.create(rc);
                    LOG.error("Couldn't delete ledger {} from bookkeeper for {} : ", new Object[] { ledgerMetadata.getLedgerId(), getFullyQualifiedName(), bke });
                    promise.setException(bke);
                    return;
                }
                // after the ledger is deleted, we delete the metadata znode
                scheduler.submit(new Runnable() {

                    @Override
                    public void run() {
                        deleteLogSegmentMetadata(ledgerMetadata, promise);
                    }
                });
            }
        }, null);
    } catch (IOException e) {
        promise.setException(BKException.create(BKException.Code.BookieHandleNotAvailableException));
    }
    return promise;
}