Examples with OperatorBackPressureStats org.apache.flink.runtime.webmonitor.OperatorBackPressureStats used on opensource projects

Example 1 with OperatorBackPressureStats

use of org.apache.flink.runtime.webmonitor.OperatorBackPressureStats in project flink by apache.

the class JobVertexBackPressureHandler method handleRequest.

@Override
public String handleRequest(AccessExecutionJobVertex accessJobVertex, Map<String, String> params) throws Exception {
    if (accessJobVertex instanceof ArchivedExecutionJobVertex) {
        return "";
    }
    ExecutionJobVertex jobVertex = (ExecutionJobVertex) accessJobVertex;
    try (StringWriter writer = new StringWriter();
        JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer)) {
        gen.writeStartObject();
        Option<OperatorBackPressureStats> statsOption = backPressureStatsTracker.getOperatorBackPressureStats(jobVertex);
        if (statsOption.isDefined()) {
            OperatorBackPressureStats stats = statsOption.get();
            // Check whether we need to refresh
            if (refreshInterval <= System.currentTimeMillis() - stats.getEndTimestamp()) {
                backPressureStatsTracker.triggerStackTraceSample(jobVertex);
                gen.writeStringField("status", "deprecated");
            } else {
                gen.writeStringField("status", "ok");
            }
            gen.writeStringField("backpressure-level", getBackPressureLevel(stats.getMaxBackPressureRatio()));
            gen.writeNumberField("end-timestamp", stats.getEndTimestamp());
            // Sub tasks
            gen.writeArrayFieldStart("subtasks");
            int numSubTasks = stats.getNumberOfSubTasks();
            for (int i = 0; i < numSubTasks; i++) {
                double ratio = stats.getBackPressureRatio(i);
                gen.writeStartObject();
                gen.writeNumberField("subtask", i);
                gen.writeStringField("backpressure-level", getBackPressureLevel(ratio));
                gen.writeNumberField("ratio", ratio);
                gen.writeEndObject();
            }
            gen.writeEndArray();
        } else {
            backPressureStatsTracker.triggerStackTraceSample(jobVertex);
            gen.writeStringField("status", "deprecated");
        }
        gen.writeEndObject();
        gen.close();
        return writer.toString();
    }
}

Example 2 with OperatorBackPressureStats

use of org.apache.flink.runtime.webmonitor.OperatorBackPressureStats in project flink by apache.

the class JobVertexBackPressureHandlerTest method testResponseStatsAvailable.

/** Tests the response when stats are available */
@Test
public void testResponseStatsAvailable() throws Exception {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    BackPressureStatsTracker statsTracker = mock(BackPressureStatsTracker.class);
    OperatorBackPressureStats stats = new OperatorBackPressureStats(0, System.currentTimeMillis(), new double[] { 0.31, 0.48, 1.0, 0.0 });
    when(statsTracker.getOperatorBackPressureStats(any(ExecutionJobVertex.class))).thenReturn(Option.apply(stats));
    JobVertexBackPressureHandler handler = new JobVertexBackPressureHandler(mock(ExecutionGraphHolder.class), statsTracker, 9999);
    String response = handler.handleRequest(jobVertex, Collections.<String, String>emptyMap());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(response);
    // Single element
    assertEquals(4, rootNode.size());
    // Status
    JsonNode status = rootNode.get("status");
    assertNotNull(status);
    assertEquals("ok", status.textValue());
    // Back pressure level
    JsonNode backPressureLevel = rootNode.get("backpressure-level");
    assertNotNull(backPressureLevel);
    assertEquals("high", backPressureLevel.textValue());
    // End time stamp
    JsonNode endTimeStamp = rootNode.get("end-timestamp");
    assertNotNull(endTimeStamp);
    assertEquals(stats.getEndTimestamp(), endTimeStamp.longValue());
    // Subtasks
    JsonNode subTasks = rootNode.get("subtasks");
    assertEquals(stats.getNumberOfSubTasks(), subTasks.size());
    for (int i = 0; i < subTasks.size(); i++) {
        JsonNode subTask = subTasks.get(i);
        JsonNode index = subTask.get("subtask");
        assertEquals(i, index.intValue());
        JsonNode level = subTask.get("backpressure-level");
        assertEquals(JobVertexBackPressureHandler.getBackPressureLevel(stats.getBackPressureRatio(i)), level.textValue());
        JsonNode ratio = subTask.get("ratio");
        assertEquals(stats.getBackPressureRatio(i), ratio.doubleValue(), 0.0);
    }
    // Verify not triggered
    verify(statsTracker, never()).triggerStackTraceSample(any(ExecutionJobVertex.class));
}

Example 3 with OperatorBackPressureStats

use of org.apache.flink.runtime.webmonitor.OperatorBackPressureStats in project flink by apache.

the class JobVertexBackPressureHandlerTest method testResponsePassedRefreshInterval.

/** Tests that after the refresh interval another sample is triggered. */
@Test
public void testResponsePassedRefreshInterval() throws Exception {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    BackPressureStatsTracker statsTracker = mock(BackPressureStatsTracker.class);
    OperatorBackPressureStats stats = new OperatorBackPressureStats(0, System.currentTimeMillis(), new double[] { 0.31, 0.48, 1.0, 0.0 });
    when(statsTracker.getOperatorBackPressureStats(any(ExecutionJobVertex.class))).thenReturn(Option.apply(stats));
    JobVertexBackPressureHandler handler = new JobVertexBackPressureHandler(mock(ExecutionGraphHolder.class), statsTracker, // <----- refresh interval should fire immediately
    0);
    String response = handler.handleRequest(jobVertex, Collections.<String, String>emptyMap());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(response);
    // Single element
    assertEquals(4, rootNode.size());
    // Status
    JsonNode status = rootNode.get("status");
    assertNotNull(status);
    // Interval passed, hence deprecated
    assertEquals("deprecated", status.textValue());
    // Back pressure level
    JsonNode backPressureLevel = rootNode.get("backpressure-level");
    assertNotNull(backPressureLevel);
    assertEquals("high", backPressureLevel.textValue());
    // End time stamp
    JsonNode endTimeStamp = rootNode.get("end-timestamp");
    assertNotNull(endTimeStamp);
    assertEquals(stats.getEndTimestamp(), endTimeStamp.longValue());
    // Subtasks
    JsonNode subTasks = rootNode.get("subtasks");
    assertEquals(stats.getNumberOfSubTasks(), subTasks.size());
    for (int i = 0; i < subTasks.size(); i++) {
        JsonNode subTask = subTasks.get(i);
        JsonNode index = subTask.get("subtask");
        assertEquals(i, index.intValue());
        JsonNode level = subTask.get("backpressure-level");
        assertEquals(JobVertexBackPressureHandler.getBackPressureLevel(stats.getBackPressureRatio(i)), level.textValue());
        JsonNode ratio = subTask.get("ratio");
        assertEquals(stats.getBackPressureRatio(i), ratio.doubleValue(), 0.0);
    }
    // Verify triggered
    verify(statsTracker).triggerStackTraceSample(any(ExecutionJobVertex.class));
}