use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.
the class SpecMonitoringInfoValidatorTest method validateReturnsErrorOnMissingUrnOrType.
@Test
public void validateReturnsErrorOnMissingUrnOrType() {
MonitoringInfo testInput = MonitoringInfo.newBuilder().putLabels(MonitoringInfoConstants.Labels.NAME, "anyCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyString").setType(TypeUrns.SUM_INT64_TYPE).build();
assertTrue(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
testInput = MonitoringInfo.newBuilder().setUrn(Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "anyCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyString").build();
assertTrue(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
}
use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.
the class SpecMonitoringInfoValidatorTest method validateReturnsErrorOnInvalidMonitoringInfoLabels.
@Test
public void validateReturnsErrorOnInvalidMonitoringInfoLabels() {
MonitoringInfo testInput = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "unexpectedLabel").build();
assertTrue(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
}
use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.
the class MetricsContainerStepMapTest method testUpdateAllUpdatesUnboundedAndBoundedContainers.
@Test
public void testUpdateAllUpdatesUnboundedAndBoundedContainers() {
MetricsContainerStepMap baseMetricContainerRegistry = new MetricsContainerStepMap();
CounterCell c1 = baseMetricContainerRegistry.getContainer(STEP1).getCounter(MetricName.named("ns", "name1"));
CounterCell c2 = baseMetricContainerRegistry.getUnboundContainer().getCounter(MonitoringInfoTestUtil.testElementCountName());
c1.inc(7);
c2.inc(14);
MetricsContainerStepMap testObject = new MetricsContainerStepMap();
testObject.updateAll(baseMetricContainerRegistry);
List<MonitoringInfo> expected = new ArrayList<MonitoringInfo>();
SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
builder.setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns").setLabel(MonitoringInfoConstants.Labels.NAME, "name1");
builder.setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, STEP1);
builder.setInt64SumValue(7);
expected.add(builder.build());
expected.add(MonitoringInfoTestUtil.testElementCountMonitoringInfo(14));
ArrayList<MonitoringInfo> actual = new ArrayList<MonitoringInfo>();
for (MonitoringInfo mi : testObject.getMonitoringInfos()) {
actual.add(mi);
}
assertThat(actual, containsInAnyOrder(expected.toArray()));
}
use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.
the class FlinkMetricContainerTest method testDropUnexpectedMonitoringInfoTypes.
@Test
public void testDropUnexpectedMonitoringInfoTypes() {
MetricsContainerImpl step = container.getMetricsContainer("step");
MonitoringInfo intCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns1").setLabel(MonitoringInfoConstants.Labels.NAME, "int_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64SumValue(111).build();
MonitoringInfo doubleCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns2").setLabel(MonitoringInfoConstants.Labels.NAME, "double_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleSumValue(222).build();
MonitoringInfo intDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns3").setLabel(MonitoringInfoConstants.Labels.NAME, "int_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64DistributionValue(DistributionData.create(30, 10, 1, 5)).build();
MonitoringInfo doubleDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns4").setLabel(MonitoringInfoConstants.Labels.NAME, "double_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleDistributionValue(10, 30, 1, 5).build();
// Mock out the counter that Flink returns; the distribution gets created by
// FlinkMetricContainer, not by Flink itself, so we verify it in a different way below
SimpleCounter counter = new SimpleCounter();
when(metricGroup.counter("ns1.int_counter")).thenReturn(counter);
container.updateMetrics("step", ImmutableList.of(intCounter, doubleCounter, intDistribution, doubleDistribution));
// Flink's MetricGroup should only have asked for one counter (the integer-typed one) to be
// created (the double-typed one is dropped currently)
verify(metricGroup).counter(eq("ns1.int_counter"));
// Verify that the counter injected into flink has the right value
assertThat(counter.getCount(), is(111L));
// Verify the counter in the java SDK MetricsContainer
long count = ((CounterCell) step.tryGetCounter(MonitoringInfoMetricName.of(intCounter))).getCumulative();
assertThat(count, is(111L));
// The one Flink distribution that gets created is a FlinkDistributionGauge; here we verify its
// initial (and in this test, final) value
verify(metricGroup).gauge(eq("ns3.int_distribution"), argThat(new ArgumentMatcher<FlinkDistributionGauge>() {
@Override
public boolean matches(FlinkDistributionGauge argument) {
DistributionResult actual = ((FlinkDistributionGauge) argument).getValue();
DistributionResult expected = DistributionResult.create(30, 10, 1, 5);
return actual.equals(expected);
}
}));
// Verify that the Java SDK MetricsContainer holds the same information
DistributionData distributionData = ((DistributionCell) step.getDistribution(MonitoringInfoMetricName.of(intDistribution))).getCumulative();
assertThat(distributionData, is(DistributionData.create(30, 10, 1, 5)));
}
use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.
the class FlinkMetricContainerTest method testMonitoringInfoUpdate.
@Test
public void testMonitoringInfoUpdate() {
SimpleCounter userCounter = new SimpleCounter();
when(metricGroup.counter("ns1.metric1")).thenReturn(userCounter);
SimpleCounter pCollectionCounter = new SimpleCounter();
when(metricGroup.counter("pcoll.metric:element_count:v1")).thenReturn(pCollectionCounter);
SimpleCounter pTransformCounter = new SimpleCounter();
when(metricGroup.counter("anyPTransform.myMetric")).thenReturn(pTransformCounter);
MonitoringInfo userCountMonitoringInfo = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns1").setLabel(MonitoringInfoConstants.Labels.NAME, "metric1").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").setInt64SumValue(111).build();
assertNotNull(userCountMonitoringInfo);
MonitoringInfo pCollectionScoped = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT).setInt64SumValue(222).setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, "pcoll").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").build();
assertNotNull(pCollectionScoped);
MonitoringInfo transformScoped = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.START_BUNDLE_MSECS).setInt64SumValue(333).setLabel(MonitoringInfoConstants.Labels.NAME, "myMetric").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").build();
assertNotNull(transformScoped);
assertThat(userCounter.getCount(), is(0L));
assertThat(pCollectionCounter.getCount(), is(0L));
assertThat(pTransformCounter.getCount(), is(0L));
container.updateMetrics("step", ImmutableList.of(userCountMonitoringInfo, pCollectionScoped, transformScoped));
assertThat(userCounter.getCount(), is(111L));
assertThat(pCollectionCounter.getCount(), is(222L));
assertThat(pTransformCounter.getCount(), is(333L));
}
Aggregations