Examples with ExecutionState org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState used on opensource projects

Example 1 with ExecutionState

use of org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState in project beam by apache.

the class WorkItemStatusClient method populateMetricUpdates.

@VisibleForTesting
synchronized void populateMetricUpdates(WorkItemStatus status) {
    List<MetricUpdate> updates = new ArrayList<>();
    if (executionContext != null && executionContext.getExecutionStateTracker() != null) {
        ExecutionStateTracker tracker = executionContext.getExecutionStateTracker();
        MetricUpdate update = new MetricUpdate();
        update.setKind("internal");
        MetricStructuredName name = new MetricStructuredName();
        name.setName("state-sampler");
        update.setName(name);
        Map<String, Object> metric = new HashMap<>();
        ExecutionState state = tracker.getCurrentState();
        if (state != null) {
            metric.put("last-state-name", state.getDescription());
        }
        metric.put("num-transitions", tracker.getNumTransitions());
        metric.put("last-state-duration-ms", tracker.getMillisSinceLastTransition());
        update.setInternal(metric);
        updates.add(update);
    }
    status.setMetricUpdates(updates);
}

Example 2 with ExecutionState

use of org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState in project beam by apache.

the class WorkItemStatusClientTest method populateMetricUpdatesStateSamplerInfo.

@Test
public void populateMetricUpdatesStateSamplerInfo() throws Exception {
    // When executionContext.getExecutionStateTracker() returns non-null, we get one metric update.
    WorkItemStatus status = new WorkItemStatus();
    BatchModeExecutionContext executionContext = mock(BatchModeExecutionContext.class);
    ExecutionStateTracker executionStateTracker = mock(ExecutionStateTracker.class);
    ExecutionState executionState = mock(ExecutionState.class);
    when(executionState.getDescription()).thenReturn("stageName-systemName-some-state");
    when(executionContext.getExecutionStateTracker()).thenReturn(executionStateTracker);
    when(executionStateTracker.getMillisSinceLastTransition()).thenReturn(20L);
    when(executionStateTracker.getNumTransitions()).thenReturn(10L);
    when(executionStateTracker.getCurrentState()).thenReturn(executionState);
    statusClient.setWorker(worker, executionContext);
    statusClient.populateMetricUpdates(status);
    assertThat(status.getMetricUpdates(), hasSize(1));
    MetricUpdate update = status.getMetricUpdates().get(0);
    assertThat(update.getName().getName(), equalTo("state-sampler"));
    assertThat(update.getKind(), equalTo("internal"));
    Map<String, Object> samplerMetrics = (Map<String, Object>) update.getInternal();
    assertThat(samplerMetrics, hasEntry("last-state-name", "stageName-systemName-some-state"));
    assertThat(samplerMetrics, hasEntry("num-transitions", 10L));
    assertThat(samplerMetrics, hasEntry("last-state-duration-ms", 20L));
}

Example 3 with ExecutionState

use of org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState in project beam by apache.

the class DataflowWorkerLoggingHandler method publish.

@Override
public synchronized void publish(LogRecord record) {
    DataflowExecutionState currrentDataflowState = null;
    ExecutionState currrentState = ExecutionStateTracker.getCurrentExecutionState();
    if (currrentState instanceof DataflowExecutionState) {
        currrentDataflowState = (DataflowExecutionState) currrentState;
    }
    // It's okay to pass in the null state, publish() handles and tests this.
    publish(currrentDataflowState, record);
}

Example 4 with ExecutionState

use of org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState in project beam by apache.

the class BatchModeExecutionContextTest method extractMsecCounters.

@Test
public void extractMsecCounters() {
    BatchModeExecutionContext executionContext = BatchModeExecutionContext.forTesting(PipelineOptionsFactory.create(), "testStage");
    MetricsContainer metricsContainer = Mockito.mock(MetricsContainer.class);
    ProfileScope profileScope = Mockito.mock(ProfileScope.class);
    ExecutionState start1 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-1", "system-1", "user-1"), ExecutionStateTracker.START_STATE_NAME, metricsContainer, profileScope);
    ExecutionState process1 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-1", "system-1", "user-1"), ExecutionStateTracker.PROCESS_STATE_NAME, metricsContainer, profileScope);
    ExecutionState start2 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-2", "system-2", "user-2"), ExecutionStateTracker.START_STATE_NAME, metricsContainer, profileScope);
    ExecutionState other = executionContext.executionStateRegistry.getState(NameContext.forStage("stage"), "other", null, NoopProfileScope.NOOP);
    other.takeSample(120);
    start1.takeSample(100);
    process1.takeSample(500);
    assertThat(executionContext.extractMsecCounters(false), containsInAnyOrder(msecStage("other-msecs", "stage", 120), msec("start-msecs", "stage", "original-1", 100), msec("process-msecs", "stage", "original-1", 500)));
    process1.takeSample(200);
    start2.takeSample(200);
    assertThat(executionContext.extractMsecCounters(false), containsInAnyOrder(msec("process-msecs", "stage", "original-1", 500 + 200), msec("start-msecs", "stage", "original-2", 200)));
    process1.takeSample(300);
    assertThat(executionContext.extractMsecCounters(true), hasItems(msecStage("other-msecs", "stage", 120), msec("start-msecs", "stage", "original-1", 100), msec("process-msecs", "stage", "original-1", 500 + 200 + 300), msec("start-msecs", "stage", "original-2", 200)));
}

Example 5 with ExecutionState

use of org.apache.beam.runners.core.metrics.ExecutionStateTracker.ExecutionState in project beam by apache.

the class StreamingModeExecutionContextTest method extractMsecCounters.

@Test
public void extractMsecCounters() {
    MetricsContainer metricsContainer = Mockito.mock(MetricsContainer.class);
    ProfileScope profileScope = Mockito.mock(ProfileScope.class);
    ExecutionState start1 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-1", "system-1", "user-1"), ExecutionStateTracker.START_STATE_NAME, metricsContainer, profileScope);
    ExecutionState process1 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-1", "system-1", "user-1"), ExecutionStateTracker.PROCESS_STATE_NAME, metricsContainer, profileScope);
    ExecutionState start2 = executionContext.executionStateRegistry.getState(NameContext.create("stage", "original-2", "system-2", "user-2"), ExecutionStateTracker.START_STATE_NAME, metricsContainer, profileScope);
    ExecutionState other = executionContext.executionStateRegistry.getState(NameContext.forStage("stage"), "other", null, NoopProfileScope.NOOP);
    other.takeSample(120);
    start1.takeSample(100);
    process1.takeSample(500);
    assertThat(executionStateRegistry.extractUpdates(false), containsInAnyOrder(msecStage("other-msecs", "stage", 120), msec("start-msecs", "stage", "original-1", 100), msec("process-msecs", "stage", "original-1", 500)));
    process1.takeSample(200);
    start2.takeSample(200);
    assertThat(executionStateRegistry.extractUpdates(false), containsInAnyOrder(msec("process-msecs", "stage", "original-1", 200), msec("start-msecs", "stage", "original-2", 200)));
    process1.takeSample(300);
    assertThat(executionStateRegistry.extractUpdates(false), containsInAnyOrder(msec("process-msecs", "stage", "original-1", 300)));
}