Examples with Stopwatch com.google.common.base.Stopwatch used on opensource projects

Example 6 with Stopwatch

use of com.google.common.base.Stopwatch in project hive by apache.

the class CoreCliDriver method runTest.

@Override
public void runTest(String tname, String fname, String fpath) throws Exception {
    Stopwatch sw = new Stopwatch().start();
    boolean skipped = false;
    boolean failed = false;
    try {
        LOG.info("Begin query: " + fname);
        System.err.println("Begin query: " + fname);
        qt.addFile(fpath);
        if (qt.shouldBeSkipped(fname)) {
            LOG.info("Test " + fname + " skipped");
            System.err.println("Test " + fname + " skipped");
            skipped = true;
            return;
        }
        qt.cliInit(fname, false);
        int ecode = qt.executeClient(fname);
        if (ecode != 0) {
            failed = true;
            qt.failed(ecode, fname, debugHint);
        }
        ecode = qt.checkCliDriverResults(fname);
        if (ecode != 0) {
            failed = true;
            qt.failedDiff(ecode, fname, debugHint);
        }
    } catch (Throwable e) {
        failed = true;
        qt.failed(e, fname, debugHint);
    } finally {
        String message = "Done query" + fname + ". succeeded=" + !failed + ", skipped=" + skipped + ". ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS);
        LOG.info(message);
        System.err.println(message);
    }
    assertTrue("Test passed", true);
}

Example 7 with Stopwatch

use of com.google.common.base.Stopwatch in project druid by druid-io.

the class AppenderatorImpl method persistAll.

@Override
public ListenableFuture<Object> persistAll(final Committer committer) {
    // Submit persistAll task to the persistExecutor
    final Map<SegmentIdentifier, Integer> commitHydrants = Maps.newHashMap();
    final List<Pair<FireHydrant, SegmentIdentifier>> indexesToPersist = Lists.newArrayList();
    final Set<SegmentIdentifier> identifiers = sinks.keySet();
    for (SegmentIdentifier identifier : identifiers) {
        final Sink sink = sinks.get(identifier);
        final List<FireHydrant> hydrants = Lists.newArrayList(sink);
        commitHydrants.put(identifier, hydrants.size());
        final int limit = sink.isWritable() ? hydrants.size() - 1 : hydrants.size();
        for (FireHydrant hydrant : hydrants.subList(0, limit)) {
            if (!hydrant.hasSwapped()) {
                log.info("Hydrant[%s] hasn't persisted yet, persisting. Segment[%s]", hydrant, identifier);
                indexesToPersist.add(Pair.of(hydrant, identifier));
            }
        }
        if (sink.swappable()) {
            indexesToPersist.add(Pair.of(sink.swap(), identifier));
        }
    }
    log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource());
    final String threadName = String.format("%s-incremental-persist", schema.getDataSource());
    final Object commitMetadata = committer.getMetadata();
    final Stopwatch runExecStopwatch = Stopwatch.createStarted();
    final Stopwatch persistStopwatch = Stopwatch.createStarted();
    final ListenableFuture<Object> future = persistExecutor.submit(new ThreadRenamingCallable<Object>(threadName) {

        @Override
        public Object doCall() {
            try {
                for (Pair<FireHydrant, SegmentIdentifier> pair : indexesToPersist) {
                    metrics.incrementRowOutputCount(persistHydrant(pair.lhs, pair.rhs));
                }
                log.info("Committing metadata[%s] for sinks[%s].", commitMetadata, Joiner.on(", ").join(Iterables.transform(commitHydrants.entrySet(), new Function<Map.Entry<SegmentIdentifier, Integer>, String>() {

                    @Override
                    public String apply(Map.Entry<SegmentIdentifier, Integer> entry) {
                        return String.format("%s:%d", entry.getKey().getIdentifierAsString(), entry.getValue());
                    }
                })));
                committer.run();
                objectMapper.writeValue(computeCommitFile(), Committed.create(commitHydrants, commitMetadata));
                return commitMetadata;
            } catch (Exception e) {
                metrics.incrementFailedPersists();
                throw Throwables.propagate(e);
            } finally {
                metrics.incrementNumPersists();
                metrics.incrementPersistTimeMillis(persistStopwatch.elapsed(TimeUnit.MILLISECONDS));
                persistStopwatch.stop();
            }
        }
    });
    final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS);
    metrics.incrementPersistBackPressureMillis(startDelay);
    if (startDelay > WARN_DELAY) {
        log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay);
    }
    runExecStopwatch.stop();
    resetNextFlush();
    // NB: The rows are still in memory until they're done persisting, but we only count rows in active indexes.
    rowsCurrentlyInMemory.set(0);
    return future;
}

Example 8 with Stopwatch

use of com.google.common.base.Stopwatch in project druid by druid-io.

the class RealtimePlumber method persist.

@Override
public void persist(final Committer committer) {
    final List<Pair<FireHydrant, Interval>> indexesToPersist = Lists.newArrayList();
    for (Sink sink : sinks.values()) {
        if (sink.swappable()) {
            indexesToPersist.add(Pair.of(sink.swap(), sink.getInterval()));
        }
    }
    log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource());
    final Stopwatch runExecStopwatch = Stopwatch.createStarted();
    final Stopwatch persistStopwatch = Stopwatch.createStarted();
    final Map<String, Object> metadataElems = committer.getMetadata() == null ? null : ImmutableMap.of(COMMIT_METADATA_KEY, committer.getMetadata(), COMMIT_METADATA_TIMESTAMP_KEY, System.currentTimeMillis());
    persistExecutor.execute(new ThreadRenamingRunnable(String.format("%s-incremental-persist", schema.getDataSource())) {

        @Override
        public void doRun() {
            /* Note:
            If plumber crashes after storing a subset of all the hydrants then we will lose data and next
            time we will start with the commitMetadata stored in those hydrants.
            option#1:
            maybe it makes sense to store the metadata outside the segments in a separate file. This is because the
            commit metadata isn't really associated with an individual segment-- it's associated with a set of segments
            that are persisted at the same time or maybe whole datasource. So storing it in the segments is asking for problems.
            Sort of like this:

            {
              "metadata" : {"foo": "bar"},
              "segments": [
                {"id": "datasource_2000_2001_2000_1", "hydrant": 10},
                {"id": "datasource_2001_2002_2001_1", "hydrant": 12},
              ]
            }
            When a realtime node crashes and starts back up, it would delete any hydrants numbered higher than the
            ones in the commit file.

            option#2
            We could also just include the set of segments for the same chunk of metadata in more metadata on each
            of the segments. we might also have to think about the hand-off in terms of the full set of segments being
            handed off instead of individual segments being handed off (that is, if one of the set succeeds in handing
            off and the others fail, the real-time would believe that it needs to re-ingest the data).
             */
            long persistThreadCpuTime = VMUtils.safeGetThreadCpuTime();
            try {
                for (Pair<FireHydrant, Interval> pair : indexesToPersist) {
                    metrics.incrementRowOutputCount(persistHydrant(pair.lhs, schema, pair.rhs, metadataElems));
                }
                committer.run();
            } catch (Exception e) {
                metrics.incrementFailedPersists();
                throw e;
            } finally {
                metrics.incrementPersistCpuTime(VMUtils.safeGetThreadCpuTime() - persistThreadCpuTime);
                metrics.incrementNumPersists();
                metrics.incrementPersistTimeMillis(persistStopwatch.elapsed(TimeUnit.MILLISECONDS));
                persistStopwatch.stop();
            }
        }
    });
    final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS);
    metrics.incrementPersistBackPressureMillis(startDelay);
    if (startDelay > WARN_DELAY) {
        log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay);
    }
    runExecStopwatch.stop();
    resetNextFlush();
}

Example 9 with Stopwatch

use of com.google.common.base.Stopwatch in project druid by druid-io.

the class BatchServerInventoryViewTest method waitForUpdateEvents.

private void waitForUpdateEvents(int count) throws Exception {
    final Timing forWaitingTiming = timing.forWaiting();
    Stopwatch stopwatch = Stopwatch.createStarted();
    while (inventoryUpdateCounter.get() != count) {
        Thread.sleep(100);
        if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > forWaitingTiming.milliseconds()) {
            throw new ISE("BatchServerInventoryView is not updating counter expected[%d] value[%d]", count, inventoryUpdateCounter.get());
        }
    }
}

Example 10 with Stopwatch

use of com.google.common.base.Stopwatch in project druid by druid-io.

the class RealtimeManagerTest method testRunV2.

@Test
public void testRunV2() throws Exception {
    realtimeManager2.start();
    Stopwatch stopwatch = Stopwatch.createStarted();
    while (realtimeManager2.getMetrics("testV2").processed() != 1) {
        Thread.sleep(100);
        if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > 1000) {
            throw new ISE("Realtime manager should have completed processing 2 events!");
        }
    }
    Assert.assertEquals(1, realtimeManager2.getMetrics("testV2").processed());
    Assert.assertEquals(1, realtimeManager2.getMetrics("testV2").thrownAway());
    Assert.assertEquals(2, realtimeManager2.getMetrics("testV2").unparseable());
    Assert.assertTrue(plumber2.isStartedJob());
    Assert.assertTrue(plumber2.isFinishedJob());
    Assert.assertEquals(0, plumber2.getPersistCount());
}