Examples with NameContext org.apache.beam.runners.dataflow.worker.counters.NameContext used on opensource projects

Example 26 with NameContext

use of org.apache.beam.runners.dataflow.worker.counters.NameContext in project beam by apache.

the class DataflowElementExecutionTrackerTest method testNoExecutionsSinceLastSample.

/**
 * Ensure that sampling is properly handled when there are no new executions since the last
 * sampling period.
 */
@Test
public void testNoExecutionsSinceLastSample() throws IOException {
    // Journal: IDLE
    tracker.takeSample(10);
    NameContext step = createStep("A");
    tracker.enter(step);
    // Journal: IDLE A1
    tracker.takeSample(10);
    // Journal: A1
    tracker.takeSample(10);
    tracker.exit();
    // Journal: A1 IDLE
    tracker.takeSample(10);
    assertThat(getCounterValue(step), equalTo(distribution(20)));
    // Journal: IDLE
    tracker.takeSample(10);
    assertThat(getCounterValue(step), equalTo(distribution(20)));
}

Example 27 with NameContext

use of org.apache.beam.runners.dataflow.worker.counters.NameContext in project beam by apache.

the class DataflowElementExecutionTrackerTest method testCurrentOperationCountedInNextSample.

/**
 * Test that the currently processing element at time of sampling is also counted in the next
 * sampling period.
 */
@Test
public void testCurrentOperationCountedInNextSample() throws IOException {
    NameContext step = createStep("A");
    tracker.enter(step);
    // Journal: IDLE A1
    tracker.takeSample(20);
    // Journal: A1
    tracker.takeSample(10);
    tracker.exit();
    // Journal: A1 IDLE
    tracker.takeSample(20);
    assertThat(getCounterValue(step), equalTo(distribution(30)));
}

Example 28 with NameContext

use of org.apache.beam.runners.dataflow.worker.counters.NameContext in project beam by apache.

the class DataflowElementExecutionTrackerTest method testSampledTimeDistributedBetweenExecutionFragments.

/**
 * Test that the sampling time is distributed evenly between all execution fragments since last
 * sampling.
 */
@Test
public void testSampledTimeDistributedBetweenExecutionFragments() throws IOException {
    NameContext stepA = createStep("A");
    NameContext stepB = createStep("B");
    tracker.enter(stepA);
    tracker.exit();
    tracker.enter(stepB);
    tracker.exit();
    // Expected journal: IDLE A1 IDLE B1 IDLE
    tracker.takeSample(50);
    assertThat(getCounterValue(stepA), equalTo(distribution(10)));
    assertThat(getCounterValue(stepB), equalTo(distribution(10)));
}

Example 29 with NameContext

use of org.apache.beam.runners.dataflow.worker.counters.NameContext in project beam by apache.

the class StreamingModeExecutionContextTest method testTimerInternalsSetTimer.

@Test
public void testTimerInternalsSetTimer() {
    Windmill.WorkItemCommitRequest.Builder outputBuilder = Windmill.WorkItemCommitRequest.newBuilder();
    NameContext nameContext = NameContextsForTests.nameContextForTest();
    DataflowOperationContext operationContext = executionContext.createOperationContext(nameContext);
    StreamingModeExecutionContext.StepContext stepContext = executionContext.getStepContext(operationContext);
    executionContext.start("key", Windmill.WorkItem.newBuilder().setKey(ByteString.EMPTY).setWorkToken(17L).build(), // input watermark
    new Instant(1000), // output watermark
    null, // synchronized processing time
    null, stateReader, stateFetcher, outputBuilder);
    TimerInternals timerInternals = stepContext.timerInternals();
    timerInternals.setTimer(TimerData.of(new StateNamespaceForTest("key"), new Instant(5000), new Instant(5000), TimeDomain.EVENT_TIME));
    executionContext.flushState();
    Windmill.Timer timer = outputBuilder.buildPartial().getOutputTimers(0);
    assertThat(timer.getTag().toStringUtf8(), equalTo("/skey+0:5000"));
    assertThat(timer.getTimestamp(), equalTo(TimeUnit.MILLISECONDS.toMicros(5000)));
    assertThat(timer.getType(), equalTo(Windmill.Timer.Type.WATERMARK));
}

Example 30 with NameContext

use of org.apache.beam.runners.dataflow.worker.counters.NameContext in project beam by apache.

the class ExecutionTimeMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsValidCounterUpdateWhenValidMSecMonitoringInfoReceived.

@Test
public void testTransformReturnsValidCounterUpdateWhenValidMSecMonitoringInfoReceived() throws Exception {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    NameContext nc = NameContext.create("anyStageName", "anyOriginalName", "anySystemName", "anyUserName");
    DataflowStepContext dsc = mock(DataflowStepContext.class);
    when(dsc.getNameContext()).thenReturn(nc);
    stepContextMapping.put("anyValue", dsc);
    ExecutionTimeMonitoringInfoToCounterUpdateTransformer testObject = new ExecutionTimeMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    // Execute
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.PROCESS_BUNDLE_MSECS).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").setPayload(encodeInt64Counter(1L)).build();
    CounterUpdate result = testObject.transform(monitoringInfo);
    // Validate
    assertNotNull(result);
    assertEqualsAsJson("{cumulative:true, integer:{highBits:0, lowBits:1}, " + "structuredNameAndMetadata:{metadata:{kind:'SUM'}, " + "name:{executionStepName:'anyStageName', name:'process-msecs', origin:'SYSTEM', " + "originalStepName:'anyOriginalName'}}}", result);
}