Examples with MetricsFilter org.apache.beam.sdk.metrics.MetricsFilter used on opensource projects

Example 1 with MetricsFilter

use of org.apache.beam.sdk.metrics.MetricsFilter in project beam by apache.

the class ResumeFromCheckpointStreamingTest method testWithResume.

@Test
@Category(UsesCheckpointRecovery.class)
public void testWithResume() throws Exception {
    // write to Kafka
    produce(ImmutableMap.of("k1", new Instant(100), "k2", new Instant(200), "k3", new Instant(300), "k4", new Instant(400)));
    MetricsFilter metricsFilter = MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace(ResumeFromCheckpointStreamingTest.class)).build();
    // first run should expect EOT matching the last injected element.
    SparkPipelineResult res = run(Optional.of(new Instant(400)), 0);
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "allMessages", "EOFShallNotPassFn", 4L)));
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "processedMessages", "EOFShallNotPassFn", 4L)));
    //--- between executions:
    //- clear state.
    clean();
    //- write a bit more.
    produce(ImmutableMap.of("k5", new Instant(499), // to be dropped from [0, 500).
    "EOF", // to be dropped from [0, 500).
    new Instant(500)));
    // recovery should resume from last read offset, and read the second batch of input.
    res = runAgain(1);
    // assertions 2:
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "processedMessages", "EOFShallNotPassFn", 5L)));
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "allMessages", "EOFShallNotPassFn", 6L)));
    long successAssertions = 0;
    Iterable<MetricResult<Long>> counterResults = res.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(PAssertWithoutFlatten.class, PAssert.SUCCESS_COUNTER)).build()).counters();
    for (MetricResult<Long> counter : counterResults) {
        if (counter.attempted().longValue() > 0) {
            successAssertions++;
        }
    }
    assertThat(String.format("Expected %d successful assertions, but found %d.", 1L, successAssertions), successAssertions, is(1L));
    // validate assertion didn't fail.
    long failedAssertions = 0;
    Iterable<MetricResult<Long>> failCounterResults = res.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(PAssertWithoutFlatten.class, PAssert.FAILURE_COUNTER)).build()).counters();
    for (MetricResult<Long> counter : failCounterResults) {
        if (counter.attempted().longValue() > 0) {
            failedAssertions++;
        }
    }
    assertThat(String.format("Found %d failed assertions.", failedAssertions), failedAssertions, is(0L));
}