Examples with LongSummaryStatistics java.util.LongSummaryStatistics used on opensource projects

Example 21 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project pravega by pravega.

the class PersistentStreamBase method compareStreamCuts.

@Override
public CompletableFuture<StreamCutComparison> compareStreamCuts(Map<Long, Long> streamcut1, Map<Long, Long> streamcut2, OperationContext context) {
    Preconditions.checkNotNull(context, "operation context cannot be null");
    LongSummaryStatistics stats1 = streamcut1.keySet().stream().collect(Collectors.summarizingLong(Long::longValue));
    LongSummaryStatistics stats2 = streamcut1.keySet().stream().collect(Collectors.summarizingLong(Long::longValue));
    // streamcuts at all and can simply return the response.
    if (stats1.getMax() < stats2.getMin()) {
        // stats1 less than stats2
        return CompletableFuture.completedFuture(StreamCutComparison.Before);
    } else if (stats2.getMax() < stats1.getMin()) {
        // stats2 less than min
        return CompletableFuture.completedFuture(StreamCutComparison.EqualOrAfter);
    }
    CompletableFuture<ImmutableMap<StreamSegmentRecord, Integer>> span1Future = computeStreamCutSpan(streamcut1, context);
    CompletableFuture<ImmutableMap<StreamSegmentRecord, Integer>> span2Future = computeStreamCutSpan(streamcut2, context);
    return CompletableFuture.allOf(span1Future, span2Future).thenApply(v -> {
        ImmutableMap<StreamSegmentRecord, Integer> span1 = span1Future.join();
        ImmutableMap<StreamSegmentRecord, Integer> span2 = span2Future.join();
        // loop over all segments in streamcut1 and compare them with segments in streamcut2.
        // if we find all segments in streamcut1 greater than or equal to all segments in streamcut2
        boolean foundGt = false;
        boolean foundLt = false;
        for (Map.Entry<StreamSegmentRecord, Integer> e1 : span1.entrySet()) {
            for (Map.Entry<StreamSegmentRecord, Integer> e2 : span2.entrySet()) {
                int comparison;
                if (e2.getKey().segmentId() == e1.getKey().segmentId()) {
                    // same segment. compare offsets
                    comparison = Long.compare(streamcut1.get(e1.getKey().segmentId()), streamcut2.get(e2.getKey().segmentId()));
                } else if (e2.getKey().overlaps(e1.getKey())) {
                    // overlapping segment. compare segment id.
                    comparison = Long.compare(e1.getKey().segmentId(), e2.getKey().segmentId());
                } else {
                    continue;
                }
                foundGt = !foundGt ? comparison > 0 : foundGt;
                foundLt = !foundLt ? comparison < 0 : foundLt;
            }
        }
        if (foundGt) {
            if (foundLt) {
                // some segments are greater and some less. return overlapping
                return StreamCutComparison.Overlaps;
            } else {
                // segments are only greater or equal.
                return StreamCutComparison.EqualOrAfter;
            }
        } else {
            if (foundLt) {
                // no segment greater than but some segment less than.
                return StreamCutComparison.Before;
            } else {
                // no segment greater than no segment less than. this means all segments are equal.
                return StreamCutComparison.EqualOrAfter;
            }
        }
    });
}

Example 22 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project metron by apache.

the class ProfileBuilderBolt method log.

/**
 * Logs information about the {@link TupleWindow}.
 *
 * @param window The tuple window.
 */
private void log(TupleWindow window) {
    // summarize the newly received tuples
    LongSummaryStatistics received = window.get().stream().map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)).collect(Collectors.summarizingLong(Long::longValue));
    LOG.debug("Tuple(s) received; count={}, min={}, max={}, range={} ms", received.getCount(), received.getMin(), received.getMax(), received.getMax() - received.getMin());
    if (window.getExpired().size() > 0) {
        // summarize the expired tuples
        LongSummaryStatistics expired = window.getExpired().stream().map(tuple -> getField(TIMESTAMP_TUPLE_FIELD, tuple, Long.class)).collect(Collectors.summarizingLong(Long::longValue));
        LOG.debug("Tuple(s) expired; count={}, min={}, max={}, range={} ms, lag={} ms", expired.getCount(), expired.getMin(), expired.getMax(), expired.getMax() - expired.getMin(), received.getMin() - expired.getMin());
    }
}

Example 23 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project accumulo by apache.

the class SummaryIT method basicSummaryTest.

@Test
public void basicSummaryTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        NewTableConfiguration ntc = new NewTableConfiguration();
        SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BasicSummarizer.class.getName()).build();
        ntc.enableSummarization(sc1);
        c.tableOperations().create(table, ntc);
        BatchWriter bw = writeData(table, c);
        Collection<Summary> summaries = c.tableOperations().summaries(table).flush(false).retrieve();
        assertEquals(0, summaries.size());
        LongSummaryStatistics stats = getTimestampStats(table, c);
        summaries = c.tableOperations().summaries(table).flush(true).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        Mutation m = new Mutation(String.format("r%09x", 999));
        m.put("f1", "q1", "999-0");
        m.putDelete("f1", "q2");
        bw.addMutation(m);
        bw.flush();
        c.tableOperations().flush(table, null, null, true);
        stats = getTimestampStats(table, c);
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_002L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 1L);
        bw.close();
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // split tablet into two
        String sp1 = String.format("r%09x", 50_000);
        addSplits(table, c, sp1);
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // compact 2nd tablet
        c.tableOperations().compact(table, new CompactionConfig().setStartRow(new Text(sp1)).setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 1, TOTAL_STAT, 113_999L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // get summaries for first tablet
        stats = getTimestampStats(table, c, sp1, null);
        summaries = c.tableOperations().summaries(table).startRow(sp1).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 49_999L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // compact all tablets and regenerate all summaries
        c.tableOperations().compact(table, new CompactionConfig());
        summaries = c.tableOperations().summaries(table).retrieve();
        stats = getTimestampStats(table, c);
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        summaries = c.tableOperations().summaries(table).startRow(String.format("r%09x", 75_000)).endRow(String.format("r%09x", 80_000)).retrieve();
        Summary summary = Iterables.getOnlyElement(summaries);
        assertEquals(1, summary.getFileStatistics().getTotal());
        assertEquals(1, summary.getFileStatistics().getExtra());
        long total = summary.getStatistics().get(TOTAL_STAT);
        assertTrue("Total " + total + " out of expected range", total > 0 && total <= 10_000);
        // test adding and removing
        c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().contains("foo"));
        List<SummarizerConfiguration> summarizers = c.tableOperations().listSummarizers(table);
        assertEquals(1, summarizers.size());
        assertTrue(summarizers.contains(sc1));
        c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().equals(BasicSummarizer.class.getName()));
        summarizers = c.tableOperations().listSummarizers(table);
        assertEquals(0, summarizers.size());
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        assertEquals(0, summaries.size());
        c.tableOperations().addSummarizers(table, sc1);
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
    }
}

Example 24 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project intellij-community by JetBrains.

the class Main method test.

public static LongSummaryStatistics test(List<List<String>> list) {
    LongSummaryStatistics stat = new LongSummaryStatistics();
    for (List<String> a : list) {
        if (a != null) {
            for (String s : a) {
                long length = s.length();
                stat.accept(length);
            }
        }
    }
    return stat;
}

Example 25 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project jdk8u_jdk by JetBrains.

the class SummaryStatisticsTest method testLongStatistics.

public void testLongStatistics() {
    List<LongSummaryStatistics> instances = new ArrayList<>();
    instances.add(countTo(1000).stream().collect(Collectors.summarizingLong(i -> i)));
    instances.add(countTo(1000).stream().mapToLong(i -> i).summaryStatistics());
    instances.add(countTo(1000).parallelStream().collect(Collectors.summarizingLong(i -> i)));
    instances.add(countTo(1000).parallelStream().mapToLong(i -> i).summaryStatistics());
    for (LongSummaryStatistics stats : instances) {
        assertEquals(stats.getCount(), 1000);
        assertEquals(stats.getSum(), (long) countTo(1000).stream().mapToInt(i -> i).sum());
        assertEquals(stats.getMax(), 1000L);
        assertEquals(stats.getMin(), 1L);
    }
}