use of com.google.api.services.dataflow.model.DistributionUpdate in project beam by apache.
the class StreamingStepMetricsContainerTest method testDistributionUpdateExtraction.
@Test
public void testDistributionUpdateExtraction() {
Distribution distribution = c1.getDistribution(name1);
distribution.update(5);
distribution.update(6);
distribution.update(7);
Iterable<CounterUpdate> updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
assertThat(updates, containsInAnyOrder(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setOriginNamespace("ns").setName("name1").setOriginalStepName("s1")).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()))).setCumulative(false).setDistribution(new DistributionUpdate().setCount(longToSplitInt(3)).setMax(longToSplitInt(7)).setMin(longToSplitInt(5)).setSum(longToSplitInt(18)))));
c1.getDistribution(name1).update(3);
updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
assertThat(updates, containsInAnyOrder(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setOriginNamespace("ns").setName("name1").setOriginalStepName("s1")).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()))).setCumulative(false).setDistribution(new DistributionUpdate().setCount(longToSplitInt(1)).setMax(longToSplitInt(3)).setMin(longToSplitInt(3)).setSum(longToSplitInt(3)))));
}
use of com.google.api.services.dataflow.model.DistributionUpdate in project beam by apache.
the class DistributionCounterUpdateAggregatorTest method setUp.
@Before
public void setUp() {
counterUpdates = new ArrayList<>();
aggregator = new DistributionCounterUpdateAggregator();
for (int i = 0; i < 10; i++) {
counterUpdates.add(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.MEAN.toString()))).setDistribution(new DistributionUpdate().setSum(longToSplitInt((long) i)).setMax(longToSplitInt((long) i)).setMin(longToSplitInt((long) i)).setCount(longToSplitInt((long) 1))));
}
}
use of com.google.api.services.dataflow.model.DistributionUpdate in project beam by apache.
the class UserDistributionMonitoringInfoToCounterUpdateTransformer method transform.
/**
* Transforms user counter MonitoringInfo to relevant CounterUpdate.
*
* @return Relevant CounterUpdate or null if transformation failed.
*/
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
Optional<String> validationResult = validate(monitoringInfo);
if (validationResult.isPresent()) {
LOG.debug(validationResult.get());
return null;
}
DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
Map<String, String> miLabels = monitoringInfo.getLabelsMap();
final String ptransform = miLabels.get(MonitoringInfoConstants.Labels.PTRANSFORM);
final String counterName = miLabels.get(MonitoringInfoConstants.Labels.NAME);
final String counterNamespace = miLabels.get(MonitoringInfoConstants.Labels.NAMESPACE);
CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
DataflowStepContext stepContext = transformIdMapping.get(ptransform);
name.setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setOriginNamespace(counterNamespace)).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()));
return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setDistribution(new DistributionUpdate().setMax(DataflowCounterUpdateExtractor.longToSplitInt(data.max())).setMin(DataflowCounterUpdateExtractor.longToSplitInt(data.min())).setSum(DataflowCounterUpdateExtractor.longToSplitInt(data.sum())).setCount(DataflowCounterUpdateExtractor.longToSplitInt(data.count())));
}
use of com.google.api.services.dataflow.model.DistributionUpdate in project beam by apache.
the class MetricsToCounterUpdateConverter method fromDistribution.
public static CounterUpdate fromDistribution(MetricKey key, boolean isCumulative, DistributionData update) {
CounterStructuredNameAndMetadata name = structuredNameAndMetadata(key, Kind.DISTRIBUTION);
DistributionUpdate distributionUpdateProto = new DistributionUpdate();
distributionUpdateProto.setMin(longToSplitInt(update.min())).setMax(longToSplitInt(update.max())).setCount(longToSplitInt(update.count())).setSum(longToSplitInt(update.sum()));
return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(isCumulative).setDistribution(distributionUpdateProto);
}
use of com.google.api.services.dataflow.model.DistributionUpdate in project beam by apache.
the class BatchModeExecutionContextTest method extractMetricUpdatesDistribution.
@Test
public void extractMetricUpdatesDistribution() {
BatchModeExecutionContext executionContext = BatchModeExecutionContext.forTesting(PipelineOptionsFactory.create(), "testStage");
DataflowOperationContext operationContext = executionContext.createOperationContext(NameContextsForTests.nameContextForTest());
Distribution distribution = operationContext.metricsContainer().getDistribution(MetricName.named("namespace", "some-distribution"));
distribution.update(2);
distribution.update(8);
final CounterUpdate expected = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-distribution").setOriginalStepName("originalName")).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()))).setCumulative(true).setDistribution(new DistributionUpdate().setCount(longToSplitInt(2)).setMax(longToSplitInt(8)).setMin(longToSplitInt(2)).setSum(longToSplitInt(10)));
assertThat(executionContext.extractMetricUpdates(false), containsInAnyOrder(expected));
}
Aggregations