Examples with CounterDistribution org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution used on opensource projects

Example 1 with CounterDistribution

use of org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution in project beam by apache.

the class MapTaskExecutorTest method testPerElementProcessingTimeCounters.

/**
 * Verify counts for the per-element-output-time counter are correct.
 */
@Test
public void testPerElementProcessingTimeCounters() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    options.as(DataflowPipelineDebugOptions.class).setExperiments(Lists.newArrayList(DataflowElementExecutionTracker.TIME_PER_ELEMENT_EXPERIMENT));
    ExecutionStateSampler stateSampler = ExecutionStateSampler.newForTest();
    DataflowExecutionStateTracker stateTracker = new DataflowExecutionStateTracker(stateSampler, new TestDataflowExecutionState(NameContext.forStage("test-stage"), "other", null, /* requestingStepName */
    null, /* sideInputIndex */
    null, /* metricsContainer */
    NoopProfileScope.NOOP), counterSet, options, "test-work-item-id");
    NameContext parDoName = nameForStep("s1");
    // Wire a read operation with 3 elements to a ParDoOperation and assert that we count
    // the correct number of elements.
    ReadOperation read = ReadOperation.forTest(new TestReader("a", "b", "c"), new OutputReceiver(), TestOperationContext.create(counterSet, nameForStep("s0"), null, stateTracker));
    ParDoOperation parDo = new ParDoOperation(new NoopParDoFn(), new OutputReceiver[0], TestOperationContext.create(counterSet, parDoName, null, stateTracker));
    parDo.attachInput(read, 0);
    List<Operation> operations = Lists.newArrayList(read, parDo);
    try (MapTaskExecutor executor = new MapTaskExecutor(operations, counterSet, stateTracker)) {
        executor.execute();
    }
    stateSampler.doSampling(100L);
    CounterName counterName = CounterName.named("per-element-processing-time").withOriginalName(parDoName);
    Counter<Long, CounterDistribution> counter = (Counter<Long, CounterDistribution>) counterSet.getExistingCounter(counterName);
    assertThat(counter.getAggregate().getCount(), equalTo(3L));
}

Example 2 with CounterDistribution

use of org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution in project beam by apache.

the class SimpleParDoFnTest method testOutputsPerElementCounter.

@Test
public void testOutputsPerElementCounter() throws Exception {
    int[] inputData = new int[] { 1, 2, 3, 4, 5 };
    CounterDistribution expectedDistribution = CounterDistribution.builder().minMax(1, 5).count(5).sum(1 + 2 + 3 + 4 + 5).sumOfSquares(1 + 4 + 9 + 16 + 25).buckets(1, Lists.newArrayList(1L, 3L, 1L)).build();
    List<CounterUpdate> counterUpdates = executeParDoFnCounterTest(inputData);
    CounterName expectedName = CounterName.named("per-element-output-count").withOriginalName(stepContext.getNameContext());
    assertThat(counterUpdates, contains(allOf(hasStructuredName(expectedName, "DISTRIBUTION"), hasDistribution(expectedDistribution))));
}

Example 3 with CounterDistribution

use of org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution in project beam by apache.

the class CounterFactoryTest method testCounterDistributionAddValue.

@Test
public void testCounterDistributionAddValue() {
    CounterDistribution counter = CounterDistribution.empty();
    List<Long> expectedBuckets = ImmutableList.of(1L, 3L, 0L, 0L, 0L, 0L, 0L, 0L, 1L, 1L);
    for (long value : new long[] { 1, 500, 2, 3, 1000, 4 }) {
        counter = counter.addValue(value);
    }
    assertEquals(expectedBuckets, counter.getBuckets());
    assertEquals(1250030.0, counter.getSumOfSquares(), 0);
    assertEquals(1510, counter.getSum());
    assertEquals(1, counter.getFirstBucketOffset());
    assertEquals(6, counter.getCount());
    assertEquals(1, counter.getMin());
    assertEquals(1000, counter.getMax());
}

Example 4 with CounterDistribution

use of org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution in project beam by apache.

the class CounterTest method testDistribution.

@Test
public void testDistribution() {
    Counter<Long, CounterDistribution> c = counters.distribution(name);
    CounterDistribution expected = CounterDistribution.builder().minMax(Long.MAX_VALUE, 0L).count(0L).sum(0L).sumOfSquares(0f).buckets(0, new ArrayList<>()).build();
    assertEquals(expected, c.getAggregate());
    c.addValue(2L).addValue(10L).addValue(4L);
    expected = CounterDistribution.builder().minMax(2L, 10L).count(3).sum(2L + 10L + 4L).sumOfSquares(4L + 100L + 16L).buckets(2, Lists.newArrayList(2L, 0L, 1L)).build();
    assertEquals(expected, c.getAggregate());
    c.getAndReset();
    c.addValue(0L).addValue(0L);
    expected = CounterDistribution.builder().minMax(0L, 0L).count(2L).sum(0L).sumOfSquares(0f).buckets(0, Lists.newArrayList(2L)).build();
    assertEquals(expected, c.getAggregate());
    CounterDistribution distribution = c.getAndReset();
    assertEquals("getAndReset should return previous value", expected, distribution);
    expected = CounterDistribution.builder().minMax(Long.MAX_VALUE, 0L).count(0L).sum(0L).sumOfSquares(0f).buckets(0, new ArrayList<>()).build();
    assertEquals(expected, c.getAggregate());
}

Example 5 with CounterDistribution

use of org.apache.beam.runners.dataflow.worker.counters.CounterFactory.CounterDistribution in project beam by apache.

the class IntrinsicMapTaskExecutorTest method testPerElementProcessingTimeCounters.

/**
 * Verify counts for the per-element-output-time counter are correct.
 */
@Test
public void testPerElementProcessingTimeCounters() throws Exception {
    PipelineOptions options = PipelineOptionsFactory.create();
    options.as(DataflowPipelineDebugOptions.class).setExperiments(Lists.newArrayList(DataflowElementExecutionTracker.TIME_PER_ELEMENT_EXPERIMENT));
    DataflowExecutionStateTracker stateTracker = new DataflowExecutionStateTracker(ExecutionStateSampler.newForTest(), new TestDataflowExecutionState(NameContext.forStage("test-stage"), "other", null, /* requestingStepName */
    null, /* sideInputIndex */
    null, /* metricsContainer */
    NoopProfileScope.NOOP), counterSet, options, "test-work-item-id");
    NameContext parDoName = nameForStep("s1");
    // Wire a read operation with 3 elements to a ParDoOperation and assert that we count
    // the correct number of elements.
    ReadOperation read = ReadOperation.forTest(new TestReader("a", "b", "c"), new OutputReceiver(), TestOperationContext.create(counterSet, nameForStep("s0"), null, stateTracker));
    ParDoOperation parDo = new ParDoOperation(new NoopParDoFn(), new OutputReceiver[0], TestOperationContext.create(counterSet, parDoName, null, stateTracker));
    parDo.attachInput(read, 0);
    List<Operation> operations = Lists.newArrayList(read, parDo);
    try (IntrinsicMapTaskExecutor executor = IntrinsicMapTaskExecutor.withSharedCounterSet(operations, counterSet, stateTracker)) {
        executor.execute();
    }
    CounterName counterName = CounterName.named("per-element-processing-time").withOriginalName(parDoName);
    Counter<Long, CounterDistribution> counter = (Counter<Long, CounterDistribution>) counterSet.getExistingCounter(counterName);
    assertThat(counter.getAggregate().getCount(), equalTo(3L));
}