Examples with SubtaskStateStats org.apache.flink.runtime.checkpoint.SubtaskStateStats used on opensource projects

Example 1 with SubtaskStateStats

use of org.apache.flink.runtime.checkpoint.SubtaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method verifyTaskNode.

private static void verifyTaskNode(JsonNode taskNode, TaskStateStats task, long triggerTimestamp) {
    long duration = ThreadLocalRandom.current().nextInt(128);
    assertEquals(task.getLatestAckTimestamp(), taskNode.get("latest_ack_timestamp").asLong());
    assertEquals(task.getStateSize(), taskNode.get("state_size").asLong());
    assertEquals(task.getEndToEndDuration(task.getLatestAckTimestamp() - duration), taskNode.get("end_to_end_duration").asLong());
    assertEquals(task.getAlignmentBuffered(), taskNode.get("alignment_buffered").asLong());
    assertEquals(task.getNumberOfSubtasks(), taskNode.get("num_subtasks").asInt());
    assertEquals(task.getNumberOfAcknowledgedSubtasks(), taskNode.get("num_acknowledged_subtasks").asInt());
    TaskStateStats.TaskStateStatsSummary summary = task.getSummaryStats();
    verifyMinMaxAvgStats(summary.getStateSizeStats(), taskNode.get("summary").get("state_size"));
    verifyMinMaxAvgStats(summary.getSyncCheckpointDurationStats(), taskNode.get("summary").get("checkpoint_duration").get("sync"));
    verifyMinMaxAvgStats(summary.getAsyncCheckpointDurationStats(), taskNode.get("summary").get("checkpoint_duration").get("async"));
    verifyMinMaxAvgStats(summary.getAlignmentBufferedStats(), taskNode.get("summary").get("alignment").get("buffered"));
    verifyMinMaxAvgStats(summary.getAlignmentDurationStats(), taskNode.get("summary").get("alignment").get("duration"));
    JsonNode endToEndDurationNode = taskNode.get("summary").get("end_to_end_duration");
    assertEquals(summary.getAckTimestampStats().getMinimum() - triggerTimestamp, endToEndDurationNode.get("min").asLong());
    assertEquals(summary.getAckTimestampStats().getMaximum() - triggerTimestamp, endToEndDurationNode.get("max").asLong());
    assertEquals((long) summary.getAckTimestampStats().getAverage() - triggerTimestamp, endToEndDurationNode.get("avg").asLong());
    SubtaskStateStats[] subtasks = task.getSubtaskStats();
    Iterator<JsonNode> it = taskNode.get("subtasks").iterator();
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 0, subtasks[0]);
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 1, subtasks[1]);
    assertTrue(it.hasNext());
    verifySubtaskStats(it.next(), 2, subtasks[2]);
    assertFalse(it.hasNext());
}

Example 2 with SubtaskStateStats

use of org.apache.flink.runtime.checkpoint.SubtaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method createTaskStateStats.

private static TaskStateStats createTaskStateStats(int numAcknowledged) {
    ThreadLocalRandom rand = ThreadLocalRandom.current();
    TaskStateStats task = mock(TaskStateStats.class);
    when(task.getJobVertexId()).thenReturn(new JobVertexID());
    when(task.getLatestAckTimestamp()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getStateSize()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getEndToEndDuration(anyLong())).thenReturn(rand.nextLong(1024) + 1);
    when(task.getAlignmentBuffered()).thenReturn(rand.nextLong(1024) + 1);
    when(task.getNumberOfSubtasks()).thenReturn(rand.nextInt(1024) + 1);
    when(task.getNumberOfAcknowledgedSubtasks()).thenReturn(numAcknowledged);
    TaskStateStats.TaskStateStatsSummary summary = mock(TaskStateStats.TaskStateStatsSummary.class);
    doReturn(createMinMaxAvgStats(rand)).when(summary).getStateSizeStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAckTimestampStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAlignmentBufferedStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAlignmentDurationStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getSyncCheckpointDurationStats();
    doReturn(createMinMaxAvgStats(rand)).when(summary).getAsyncCheckpointDurationStats();
    when(task.getSummaryStats()).thenReturn(summary);
    SubtaskStateStats[] subtasks = new SubtaskStateStats[3];
    subtasks[0] = createSubtaskStats(0, rand);
    subtasks[1] = createSubtaskStats(1, rand);
    subtasks[2] = null;
    when(task.getSubtaskStats()).thenReturn(subtasks);
    return task;
}

Example 3 with SubtaskStateStats

use of org.apache.flink.runtime.checkpoint.SubtaskStateStats in project flink by apache.

the class CheckpointStatsSubtaskDetailsHandlerTest method createSubtaskStats.

private static SubtaskStateStats createSubtaskStats(int index, ThreadLocalRandom rand) {
    SubtaskStateStats subtask = mock(SubtaskStateStats.class);
    when(subtask.getSubtaskIndex()).thenReturn(index);
    when(subtask.getAckTimestamp()).thenReturn(rand.nextLong(1024));
    when(subtask.getAlignmentBuffered()).thenReturn(rand.nextLong(1024));
    when(subtask.getAlignmentDuration()).thenReturn(rand.nextLong(1024));
    when(subtask.getSyncCheckpointDuration()).thenReturn(rand.nextLong(1024));
    when(subtask.getAsyncCheckpointDuration()).thenReturn(rand.nextLong(1024));
    when(subtask.getAckTimestamp()).thenReturn(rand.nextLong(1024));
    when(subtask.getStateSize()).thenReturn(rand.nextLong(1024));
    when(subtask.getEndToEndDuration(anyLong())).thenReturn(rand.nextLong(1024));
    return subtask;
}

Example 4 with SubtaskStateStats

use of org.apache.flink.runtime.checkpoint.SubtaskStateStats in project flink by apache.

the class CheckpointStatsDetailsSubtasksHandler method createSubtaskCheckpointDetailsJson.

private static String createSubtaskCheckpointDetailsJson(AbstractCheckpointStats checkpoint, TaskStateStats taskStats) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    // Overview
    gen.writeNumberField("id", checkpoint.getCheckpointId());
    gen.writeStringField("status", checkpoint.getStatus().toString());
    gen.writeNumberField("latest_ack_timestamp", taskStats.getLatestAckTimestamp());
    gen.writeNumberField("state_size", taskStats.getStateSize());
    gen.writeNumberField("end_to_end_duration", taskStats.getEndToEndDuration(checkpoint.getTriggerTimestamp()));
    gen.writeNumberField("alignment_buffered", taskStats.getAlignmentBuffered());
    gen.writeNumberField("num_subtasks", taskStats.getNumberOfSubtasks());
    gen.writeNumberField("num_acknowledged_subtasks", taskStats.getNumberOfAcknowledgedSubtasks());
    if (taskStats.getNumberOfAcknowledgedSubtasks() > 0) {
        gen.writeObjectFieldStart("summary");
        gen.writeObjectFieldStart("state_size");
        writeMinMaxAvg(gen, taskStats.getSummaryStats().getStateSizeStats());
        gen.writeEndObject();
        gen.writeObjectFieldStart("end_to_end_duration");
        MinMaxAvgStats ackTimestampStats = taskStats.getSummaryStats().getAckTimestampStats();
        gen.writeNumberField("min", Math.max(0, ackTimestampStats.getMinimum() - checkpoint.getTriggerTimestamp()));
        gen.writeNumberField("max", Math.max(0, ackTimestampStats.getMaximum() - checkpoint.getTriggerTimestamp()));
        gen.writeNumberField("avg", Math.max(0, ackTimestampStats.getAverage() - checkpoint.getTriggerTimestamp()));
        gen.writeEndObject();
        gen.writeObjectFieldStart("checkpoint_duration");
        gen.writeObjectFieldStart("sync");
        writeMinMaxAvg(gen, taskStats.getSummaryStats().getSyncCheckpointDurationStats());
        gen.writeEndObject();
        gen.writeObjectFieldStart("async");
        writeMinMaxAvg(gen, taskStats.getSummaryStats().getAsyncCheckpointDurationStats());
        gen.writeEndObject();
        gen.writeEndObject();
        gen.writeObjectFieldStart("alignment");
        gen.writeObjectFieldStart("buffered");
        writeMinMaxAvg(gen, taskStats.getSummaryStats().getAlignmentBufferedStats());
        gen.writeEndObject();
        gen.writeObjectFieldStart("duration");
        writeMinMaxAvg(gen, taskStats.getSummaryStats().getAlignmentDurationStats());
        gen.writeEndObject();
        gen.writeEndObject();
        gen.writeEndObject();
    }
    SubtaskStateStats[] subtasks = taskStats.getSubtaskStats();
    gen.writeArrayFieldStart("subtasks");
    for (int i = 0; i < subtasks.length; i++) {
        SubtaskStateStats subtask = subtasks[i];
        gen.writeStartObject();
        gen.writeNumberField("index", i);
        if (subtask != null) {
            gen.writeStringField("status", "completed");
            gen.writeNumberField("ack_timestamp", subtask.getAckTimestamp());
            gen.writeNumberField("end_to_end_duration", subtask.getEndToEndDuration(checkpoint.getTriggerTimestamp()));
            gen.writeNumberField("state_size", subtask.getStateSize());
            gen.writeObjectFieldStart("checkpoint");
            gen.writeNumberField("sync", subtask.getSyncCheckpointDuration());
            gen.writeNumberField("async", subtask.getAsyncCheckpointDuration());
            gen.writeEndObject();
            gen.writeObjectFieldStart("alignment");
            gen.writeNumberField("buffered", subtask.getAlignmentBuffered());
            gen.writeNumberField("duration", subtask.getAlignmentDuration());
            gen.writeEndObject();
        } else {
            gen.writeStringField("status", "pending_or_failed");
        }
        gen.writeEndObject();
    }
    gen.writeEndArray();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}