Examples with TaskStateStats org.apache.flink.runtime.checkpoint.TaskStateStats used on opensource projects

Example 6 with TaskStateStats

use of org.apache.flink.runtime.checkpoint.TaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method testSubtaskRequest.

/**
	 * Tests a subtask details request.
	 */
@Test
public void testSubtaskRequest() throws Exception {
    PendingCheckpointStats checkpoint = mock(PendingCheckpointStats.class);
    when(checkpoint.getCheckpointId()).thenReturn(1992139L);
    when(checkpoint.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
    // ack timestamp = duration
    when(checkpoint.getTriggerTimestamp()).thenReturn(0L);
    TaskStateStats task = createTaskStateStats(1237);
    when(checkpoint.getTaskStateStats(any(JobVertexID.class))).thenReturn(task);
    JsonNode rootNode = triggerRequest(checkpoint);
    assertEquals(checkpoint.getCheckpointId(), rootNode.get("id").asLong());
    assertEquals(checkpoint.getStatus().toString(), rootNode.get("status").asText());
    verifyTaskNode(rootNode, task, checkpoint.getTriggerTimestamp());
}

Example 7 with TaskStateStats

use of org.apache.flink.runtime.checkpoint.TaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method testSubtaskRequestNoSummary.

/**
	 * Tests a subtask details request.
	 */
@Test
public void testSubtaskRequestNoSummary() throws Exception {
    PendingCheckpointStats checkpoint = mock(PendingCheckpointStats.class);
    when(checkpoint.getCheckpointId()).thenReturn(1992139L);
    when(checkpoint.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
    // ack timestamp = duration
    when(checkpoint.getTriggerTimestamp()).thenReturn(0L);
    // no acknowledged
    TaskStateStats task = createTaskStateStats(0);
    when(checkpoint.getTaskStateStats(any(JobVertexID.class))).thenReturn(task);
    JsonNode rootNode = triggerRequest(checkpoint);
    assertNull(rootNode.get("summary"));
}

Example 8 with TaskStateStats

use of org.apache.flink.runtime.checkpoint.TaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method verifyTaskNode.

private static void verifyTaskNode(JsonNode taskNode, TaskStateStats task, long triggerTimestamp) {
    long duration = ThreadLocalRandom.current().nextInt(128);
    assertEquals(task.getLatestAckTimestamp(), taskNode.get("latest_ack_timestamp").asLong());
    assertEquals(task.getStateSize(), taskNode.get("state_size").asLong());
    assertEquals(task.getEndToEndDuration(task.getLatestAckTimestamp() - duration), taskNode.get("end_to_end_duration").asLong());
    assertEquals(task.getAlignmentBuffered(), taskNode.get("alignment_buffered").asLong());
    assertEquals(task.getNumberOfSubtasks(), taskNode.get("num_subtasks").asInt());
    assertEquals(task.getNumberOfAcknowledgedSubtasks(), taskNode.get("num_acknowledged_subtasks").asInt());
    TaskStateStats.TaskStateStatsSummary summary = task.getSummaryStats();
    verifyMinMaxAvgStats(summary.getStateSizeStats(), taskNode.get("summary").get("state_size"));
    verifyMinMaxAvgStats(summary.getSyncCheckpointDurationStats(), taskNode.get("summary").get("checkpoint_duration").get("sync"));
    verifyMinMaxAvgStats(summary.getAsyncCheckpointDurationStats(), taskNode.get("summary").get("checkpoint_duration").get("async"));
    verifyMinMaxAvgStats(summary.getAlignmentBufferedStats(), taskNode.get("summary").get("alignment").get("buffered"));
    verifyMinMaxAvgStats(summary.getAlignmentDurationStats(), taskNode.get("summary").get("alignment").get("duration"));
    JsonNode endToEndDurationNode = taskNode.get("summary").get("end_to_end_duration");
    assertEquals(summary.getAckTimestampStats().getMinimum() - triggerTimestamp, endToEndDurationNode.get("min").asLong());
    assertEquals(summary.getAckTimestampStats().getMaximum() - triggerTimestamp, endToEndDurationNode.get("max").asLong());
    assertEquals((long) summary.getAckTimestampStats().getAverage() - triggerTimestamp, endToEndDurationNode.get("avg").asLong());
    SubtaskStateStats[] subtasks = task.getSubtaskStats();
    Iterator<JsonNode> it = taskNode.get("subtasks").iterator();
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 0, subtasks[0]);
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 1, subtasks[1]);
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 2, subtasks[2]);
    assertFalse(it.hasNext());
}

Example 9 with TaskStateStats

use of org.apache.flink.runtime.checkpoint.TaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method createTaskStateStats.

private static TaskStateStats createTaskStateStats(int numAcknowledged) {
    ThreadLocalRandom rand = ThreadLocalRandom.current();
    TaskStateStats task = mock(TaskStateStats.class);
    when(task.getJobVertexId()).thenReturn(new JobVertexID());
    when(task.getLatestAckTimestamp()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getStateSize()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getEndToEndDuration(anyLong())).thenReturn(rand.nextLong(1024) + 1);
    when(task.getAlignmentBuffered()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getNumberOfSubtasks()).thenReturn(rand.nextInt(1024) + 1);
    when(task.getNumberOfAcknowledgedSubtasks()).thenReturn(numAcknowledged);
    TaskStateStats.TaskStateStatsSummary summary = mock(TaskStateStats.TaskStateStatsSummary.class);
    doReturn(createMinMaxAvgStats(rand)).when(summary).getStateSizeStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAckTimestampStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAlignmentBufferedStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAlignmentDurationStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getSyncCheckpointDurationStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAsyncCheckpointDurationStats();
    when(task.getSummaryStats()).thenReturn(summary);
    SubtaskStateStats[] subtasks = new SubtaskStateStats[3];
    subtasks[0] = createSubtaskStats(0, rand);
    subtasks[1] = createSubtaskStats(1, rand);
    subtasks[2] = null;
    when(task.getSubtaskStats()).thenReturn(subtasks);
    return task;
}

Example 10 with TaskStateStats

use of org.apache.flink.runtime.checkpoint.TaskStateStats in project flink by apache.

the class CheckpointStatsDetailsHandler method createCheckpointDetailsJson.

public static String createCheckpointDetailsJson(AbstractCheckpointStats checkpoint) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    gen.writeNumberField("id", checkpoint.getCheckpointId());
    gen.writeStringField("status", checkpoint.getStatus().toString());
    gen.writeBooleanField("is_savepoint", checkpoint.getProperties().isSavepoint());
    gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp());
    gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp());
    gen.writeNumberField("state_size", checkpoint.getStateSize());
    gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration());
    gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered());
    gen.writeNumberField("num_subtasks", checkpoint.getNumberOfSubtasks());
    gen.writeNumberField("num_acknowledged_subtasks", checkpoint.getNumberOfAcknowledgedSubtasks());
    if (checkpoint.getStatus().isCompleted()) {
        // --- Completed ---
        CompletedCheckpointStats completed = (CompletedCheckpointStats) checkpoint;
        String externalPath = completed.getExternalPath();
        if (externalPath != null) {
            gen.writeStringField("external_path", externalPath);
        }
        gen.writeBooleanField("discarded", completed.isDiscarded());
    } else if (checkpoint.getStatus().isFailed()) {
        // --- Failed ---
        FailedCheckpointStats failed = (FailedCheckpointStats) checkpoint;
        gen.writeNumberField("failure_timestamp", failed.getFailureTimestamp());
        String failureMsg = failed.getFailureMessage();
        if (failureMsg != null) {
            gen.writeStringField("failure_message", failureMsg);
        }
    }
    gen.writeObjectFieldStart("tasks");
    for (TaskStateStats taskStats : checkpoint.getAllTaskStateStats()) {
        gen.writeObjectFieldStart(taskStats.getJobVertexId().toString());
        gen.writeNumberField("latest_ack_timestamp", taskStats.getLatestAckTimestamp());
        gen.writeNumberField("state_size", taskStats.getStateSize());
        gen.writeNumberField("end_to_end_duration", taskStats.getEndToEndDuration(checkpoint.getTriggerTimestamp()));
        gen.writeNumberField("alignment_buffered", taskStats.getAlignmentBuffered());
        gen.writeNumberField("num_subtasks", taskStats.getNumberOfSubtasks());
        gen.writeNumberField("num_acknowledged_subtasks", taskStats.getNumberOfAcknowledgedSubtasks());
        gen.writeEndObject();
    }
    gen.writeEndObject();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}