use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.
the class BatchModeExecutionContextTest method extractMetricUpdatesCounter.
@Test
public void extractMetricUpdatesCounter() {
BatchModeExecutionContext executionContext = BatchModeExecutionContext.forTesting(PipelineOptionsFactory.create(), "testStage");
DataflowOperationContext operationContext = executionContext.createOperationContext(NameContextsForTests.nameContextForTest());
Counter counter = operationContext.metricsContainer().getCounter(MetricName.named("namespace", "some-counter"));
counter.inc(1);
counter.inc(41);
counter.inc(1);
counter.inc(-1);
final CounterUpdate expected = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("some-counter").setOriginalStepName("originalName")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(longToSplitInt(42));
assertThat(executionContext.extractMetricUpdates(false), containsInAnyOrder(expected));
executionContext.commitMetricUpdates();
Counter counterUncommitted = operationContext.metricsContainer().getCounter(MetricName.named("namespace", "uncommitted-counter"));
counterUncommitted.inc(64);
final CounterUpdate expectedUncommitted = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin("USER").setOriginNamespace("namespace").setName("uncommitted-counter").setOriginalStepName("originalName")).setMetadata(new CounterMetadata().setKind(Kind.SUM.toString()))).setCumulative(true).setInteger(longToSplitInt(64));
// Expect to get only the uncommitted metric, unless final update.
assertThat(executionContext.extractMetricUpdates(false), containsInAnyOrder(expectedUncommitted));
assertThat(executionContext.extractMetricUpdates(true), containsInAnyOrder(expected, expectedUncommitted));
executionContext.commitMetricUpdates();
// All Metrics are committed, expect none unless final update.
assertThat(executionContext.extractMetricUpdates(false), emptyIterable());
assertThat(executionContext.extractMetricUpdates(true), containsInAnyOrder(expected, expectedUncommitted));
}
use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.
the class CounterShortIdCacheTest method createMetricUpdateStructuredName.
private CounterUpdate createMetricUpdateStructuredName(final String name) {
CounterUpdate metricUpdate = new CounterUpdate();
metricUpdate.setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setName(name)));
return metricUpdate;
}
use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.
the class StreamingStepMetricsContainerTest method testDistributionUpdateExtraction.
@Test
public void testDistributionUpdateExtraction() {
Distribution distribution = c1.getDistribution(name1);
distribution.update(5);
distribution.update(6);
distribution.update(7);
Iterable<CounterUpdate> updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
assertThat(updates, containsInAnyOrder(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setOriginNamespace("ns").setName("name1").setOriginalStepName("s1")).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()))).setCumulative(false).setDistribution(new DistributionUpdate().setCount(longToSplitInt(3)).setMax(longToSplitInt(7)).setMin(longToSplitInt(5)).setSum(longToSplitInt(18)))));
c1.getDistribution(name1).update(3);
updates = StreamingStepMetricsContainer.extractMetricUpdates(registry);
assertThat(updates, containsInAnyOrder(new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setOriginNamespace("ns").setName("name1").setOriginalStepName("s1")).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()))).setCumulative(false).setDistribution(new DistributionUpdate().setCount(longToSplitInt(1)).setMax(longToSplitInt(3)).setMin(longToSplitInt(3)).setSum(longToSplitInt(3)))));
}
use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.
the class IsmSideInputReaderTest method testIterableSideInputReadCounter.
@Test
public void testIterableSideInputReadCounter() throws Exception {
// These are the expected msec and byte counters:
CounterUpdate expectedSideInputMsecUpdate = new CounterUpdate().setStructuredNameAndMetadata(new CounterStructuredNameAndMetadata().setMetadata(new CounterMetadata().setKind(Kind.SUM.toString())).setName(new CounterStructuredName().setOrigin("SYSTEM").setName("read-sideinput-msecs").setOriginalStepName("originalName").setExecutionStepName("stageName").setOriginalRequestingStepName("originalName2").setInputIndex(1))).setCumulative(true).setInteger(new SplitInt64().setHighBits(0).setLowBits(0L));
CounterName expectedCounterName = CounterName.named("read-sideinput-byte-count").withOriginalName(operationContext.nameContext()).withOrigin("SYSTEM").withOriginalRequestingStepName("originalName2").withInputIndex(1);
// Test startup:
Coder<WindowedValue<Long>> valueCoder = WindowedValue.getFullCoder(VarLongCoder.of(), GLOBAL_WINDOW_CODER);
IsmRecordCoder<WindowedValue<Long>> ismCoder = IsmRecordCoder.of(1, 0, ImmutableList.of(GLOBAL_WINDOW_CODER, BigEndianLongCoder.of()), valueCoder);
// Create a new state, which represents a step that receives the side input.
DataflowExecutionState state2 = executionContext.getExecutionStateRegistry().getState(NameContext.create("stageName", "originalName2", "systemName2", "userName2"), "process", null, NoopProfileScope.NOOP);
final List<KV<Long, WindowedValue<Long>>> firstElements = Arrays.asList(KV.of(0L, valueInGlobalWindow(0L)));
final List<KV<Long, WindowedValue<Long>>> secondElements = new ArrayList<>();
for (long i = 0; i < 100; i++) {
secondElements.add(KV.of(i, valueInGlobalWindow(i * 10)));
}
final PCollectionView<Iterable<Long>> view = Pipeline.create().apply(Create.empty(VarLongCoder.of())).apply(View.asIterable());
Source sourceA = initInputFile(fromKvsForList(firstElements), ismCoder);
Source sourceB = initInputFile(fromKvsForList(secondElements), ismCoder);
try (Closeable state2Closeable = executionContext.getExecutionStateTracker().enterState(state2)) {
final IsmSideInputReader reader = serialSideInputReader(view.getTagInternal().getId(), sourceA, sourceB);
// Store a strong reference to the returned value so that the logical reference
// cache is not cleared for this test.
Iterable<Long> value = reader.get(view, GlobalWindow.INSTANCE);
verifyIterable(toValueList(concat(firstElements, secondElements)), value);
// Assert that the same value reference was returned showing that it was cached.
assertSame(reader.get(view, GlobalWindow.INSTANCE), value);
Iterable<CounterUpdate> counterUpdates = executionContext.getExecutionStateRegistry().extractUpdates(true);
assertThat(counterUpdates, hasItem(expectedSideInputMsecUpdate));
Counter<?, ?> expectedCounter = counterFactory.getExistingCounter(expectedCounterName);
assertNotNull(expectedCounter);
}
}
use of com.google.api.services.dataflow.model.CounterStructuredNameAndMetadata in project beam by apache.
the class MetricsToCounterUpdateConverter method structuredNameAndMetadata.
private static CounterStructuredNameAndMetadata structuredNameAndMetadata(MetricKey metricKey, Kind kind) {
MetricName metricName = metricKey.metricName();
CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
name.setMetadata(new CounterMetadata().setKind(kind.toString()));
name.setName(new CounterStructuredName().setName(metricName.getName()).setOriginalStepName(metricKey.stepName()).setOrigin(Origin.USER.toString()).setOriginNamespace(metricName.getNamespace()));
return name;
}
Aggregations