Examples with FailedCheckpointStats org.apache.flink.runtime.checkpoint.FailedCheckpointStats used on opensource projects

Example 1 with FailedCheckpointStats

use of org.apache.flink.runtime.checkpoint.FailedCheckpointStats in project flink by apache.

the class CheckpointStatsDetailsHandlerTest method createFailedCheckpoint.

private static FailedCheckpointStats createFailedCheckpoint() {
    FailedCheckpointStats checkpoint = mock(FailedCheckpointStats.class);
    when(checkpoint.getCheckpointId()).thenReturn(1818214L);
    when(checkpoint.getStatus()).thenReturn(CheckpointStatsStatus.FAILED);
    when(checkpoint.getProperties()).thenReturn(CheckpointProperties.forStandardSavepoint());
    when(checkpoint.getTriggerTimestamp()).thenReturn(1818L);
    when(checkpoint.getLatestAckTimestamp()).thenReturn(11029222L);
    when(checkpoint.getStateSize()).thenReturn(925281L);
    when(checkpoint.getEndToEndDuration()).thenReturn(181819L);
    when(checkpoint.getAlignmentBuffered()).thenReturn(1010198L);
    when(checkpoint.getNumberOfSubtasks()).thenReturn(181271);
    when(checkpoint.getNumberOfAcknowledgedSubtasks()).thenReturn(29821);
    when(checkpoint.getFailureTimestamp()).thenReturn(123012890312093L);
    when(checkpoint.getFailureMessage()).thenReturn("failure-message");
    List<TaskStateStats> taskStats = new ArrayList<>();
    TaskStateStats task1 = createTaskStateStats();
    TaskStateStats task2 = createTaskStateStats();
    taskStats.add(task1);
    taskStats.add(task2);
    when(checkpoint.getAllTaskStateStats()).thenReturn(taskStats);
    return checkpoint;
}

Example 2 with FailedCheckpointStats

use of org.apache.flink.runtime.checkpoint.FailedCheckpointStats in project flink by apache.

the class CheckpointStatsDetailsHandlerTest method testCheckpointDetailsRequestFailedCheckpoint.

/**
	 * Tests a checkpoint details request for a failed checkpoint.
	 */
@Test
public void testCheckpointDetailsRequestFailedCheckpoint() throws Exception {
    FailedCheckpointStats checkpoint = createFailedCheckpoint();
    JsonNode rootNode = triggerRequest(checkpoint);
    compareFailedCheckpoint(checkpoint, rootNode);
    verifyTaskNodes(checkpoint.getAllTaskStateStats(), rootNode);
}

Example 3 with FailedCheckpointStats

use of org.apache.flink.runtime.checkpoint.FailedCheckpointStats in project flink by apache.

the class CheckpointStatsHandlerTest method compareCheckpointStats.

private static void compareCheckpointStats(TestCheckpointStats checkpointStats, JsonNode rootNode) {
    CheckpointStatsCounts counts = checkpointStats.counts;
    JsonNode countNode = rootNode.get("counts");
    assertEquals(counts.getNumberOfRestoredCheckpoints(), countNode.get("restored").asLong());
    assertEquals(counts.getTotalNumberOfCheckpoints(), countNode.get("total").asLong());
    assertEquals(counts.getNumberOfInProgressCheckpoints(), countNode.get("in_progress").asLong());
    assertEquals(counts.getNumberOfCompletedCheckpoints(), countNode.get("completed").asLong());
    assertEquals(counts.getNumberOfFailedCheckpoints(), countNode.get("failed").asLong());
    MinMaxAvgStats stateSizeSummary = checkpointStats.stateSizeSummary;
    JsonNode summaryNode = rootNode.get("summary");
    JsonNode sizeSummaryNode = summaryNode.get("state_size");
    assertEquals(stateSizeSummary.getMinimum(), sizeSummaryNode.get("min").asLong());
    assertEquals(stateSizeSummary.getMaximum(), sizeSummaryNode.get("max").asLong());
    assertEquals(stateSizeSummary.getAverage(), sizeSummaryNode.get("avg").asLong());
    MinMaxAvgStats durationSummary = checkpointStats.durationSummary;
    JsonNode durationSummaryNode = summaryNode.get("end_to_end_duration");
    assertEquals(durationSummary.getMinimum(), durationSummaryNode.get("min").asLong());
    assertEquals(durationSummary.getMaximum(), durationSummaryNode.get("max").asLong());
    assertEquals(durationSummary.getAverage(), durationSummaryNode.get("avg").asLong());
    MinMaxAvgStats alignmentBufferedSummary = checkpointStats.alignmentBufferedSummary;
    JsonNode alignmentBufferedNode = summaryNode.get("alignment_buffered");
    assertEquals(alignmentBufferedSummary.getMinimum(), alignmentBufferedNode.get("min").asLong());
    assertEquals(alignmentBufferedSummary.getMaximum(), alignmentBufferedNode.get("max").asLong());
    assertEquals(alignmentBufferedSummary.getAverage(), alignmentBufferedNode.get("avg").asLong());
    CompletedCheckpointStats latestCompleted = checkpointStats.latestCompleted;
    JsonNode latestNode = rootNode.get("latest");
    JsonNode latestCheckpointNode = latestNode.get("completed");
    assertEquals(latestCompleted.getCheckpointId(), latestCheckpointNode.get("id").asLong());
    assertEquals(latestCompleted.getTriggerTimestamp(), latestCheckpointNode.get("trigger_timestamp").asLong());
    assertEquals(latestCompleted.getLatestAckTimestamp(), latestCheckpointNode.get("latest_ack_timestamp").asLong());
    assertEquals(latestCompleted.getStateSize(), latestCheckpointNode.get("state_size").asLong());
    assertEquals(latestCompleted.getEndToEndDuration(), latestCheckpointNode.get("end_to_end_duration").asLong());
    assertEquals(latestCompleted.getAlignmentBuffered(), latestCheckpointNode.get("alignment_buffered").asLong());
    assertEquals(latestCompleted.getExternalPath(), latestCheckpointNode.get("external_path").asText());
    CompletedCheckpointStats latestSavepoint = checkpointStats.latestSavepoint;
    JsonNode latestSavepointNode = latestNode.get("savepoint");
    assertEquals(latestSavepoint.getCheckpointId(), latestSavepointNode.get("id").asLong());
    assertEquals(latestSavepoint.getTriggerTimestamp(), latestSavepointNode.get("trigger_timestamp").asLong());
    assertEquals(latestSavepoint.getLatestAckTimestamp(), latestSavepointNode.get("latest_ack_timestamp").asLong());
    assertEquals(latestSavepoint.getStateSize(), latestSavepointNode.get("state_size").asLong());
    assertEquals(latestSavepoint.getEndToEndDuration(), latestSavepointNode.get("end_to_end_duration").asLong());
    assertEquals(latestSavepoint.getAlignmentBuffered(), latestSavepointNode.get("alignment_buffered").asLong());
    assertEquals(latestSavepoint.getExternalPath(), latestSavepointNode.get("external_path").asText());
    FailedCheckpointStats latestFailed = checkpointStats.latestFailed;
    JsonNode latestFailedNode = latestNode.get("failed");
    assertEquals(latestFailed.getCheckpointId(), latestFailedNode.get("id").asLong());
    assertEquals(latestFailed.getTriggerTimestamp(), latestFailedNode.get("trigger_timestamp").asLong());
    assertEquals(latestFailed.getLatestAckTimestamp(), latestFailedNode.get("latest_ack_timestamp").asLong());
    assertEquals(latestFailed.getStateSize(), latestFailedNode.get("state_size").asLong());
    assertEquals(latestFailed.getEndToEndDuration(), latestFailedNode.get("end_to_end_duration").asLong());
    assertEquals(latestFailed.getAlignmentBuffered(), latestFailedNode.get("alignment_buffered").asLong());
    assertEquals(latestFailed.getFailureTimestamp(), latestFailedNode.get("failure_timestamp").asLong());
    assertEquals(latestFailed.getFailureMessage(), latestFailedNode.get("failure_message").asText());
    RestoredCheckpointStats latestRestored = checkpointStats.latestRestored;
    JsonNode latestRestoredNode = latestNode.get("restored");
    assertEquals(latestRestored.getCheckpointId(), latestRestoredNode.get("id").asLong());
    assertEquals(latestRestored.getRestoreTimestamp(), latestRestoredNode.get("restore_timestamp").asLong());
    assertEquals(latestRestored.getProperties().isSavepoint(), latestRestoredNode.get("is_savepoint").asBoolean());
    assertEquals(latestRestored.getExternalPath(), latestRestoredNode.get("external_path").asText());
    JsonNode historyNode = rootNode.get("history");
    Iterator<JsonNode> it = historyNode.iterator();
    assertTrue(it.hasNext());
    JsonNode inProgressNode = it.next();
    PendingCheckpointStats inProgress = checkpointStats.inProgress;
    compareInProgressCheckpoint(inProgress, inProgressNode);
    assertTrue(it.hasNext());
    JsonNode completedSavepointNode = it.next();
    CompletedCheckpointStats completedSavepoint = checkpointStats.completedSavepoint;
    compareCompletedSavepoint(completedSavepoint, completedSavepointNode);
    assertTrue(it.hasNext());
    JsonNode failedNode = it.next();
    FailedCheckpointStats failed = checkpointStats.failed;
    compareFailedCheckpoint(failed, failedNode);
    assertFalse(it.hasNext());
}

Example 4 with FailedCheckpointStats

use of org.apache.flink.runtime.checkpoint.FailedCheckpointStats in project flink by apache.

the class CheckpointStatsHandlerTest method createTestCheckpointStats.

private static TestCheckpointStats createTestCheckpointStats() {
    // Counts
    CheckpointStatsCounts counts = mock(CheckpointStatsCounts.class);
    when(counts.getNumberOfRestoredCheckpoints()).thenReturn(123123123L);
    when(counts.getTotalNumberOfCheckpoints()).thenReturn(12981231203L);
    when(counts.getNumberOfInProgressCheckpoints()).thenReturn(191919);
    when(counts.getNumberOfCompletedCheckpoints()).thenReturn(882828200L);
    when(counts.getNumberOfFailedCheckpoints()).thenReturn(99171510L);
    // Summary
    CompletedCheckpointStatsSummary summary = mock(CompletedCheckpointStatsSummary.class);
    MinMaxAvgStats stateSizeSummary = mock(MinMaxAvgStats.class);
    when(stateSizeSummary.getMinimum()).thenReturn(81238123L);
    when(stateSizeSummary.getMaximum()).thenReturn(19919191999L);
    when(stateSizeSummary.getAverage()).thenReturn(1133L);
    MinMaxAvgStats durationSummary = mock(MinMaxAvgStats.class);
    when(durationSummary.getMinimum()).thenReturn(1182L);
    when(durationSummary.getMaximum()).thenReturn(88654L);
    when(durationSummary.getAverage()).thenReturn(171L);
    MinMaxAvgStats alignmentBufferedSummary = mock(MinMaxAvgStats.class);
    when(alignmentBufferedSummary.getMinimum()).thenReturn(81818181899L);
    when(alignmentBufferedSummary.getMaximum()).thenReturn(89999911118654L);
    when(alignmentBufferedSummary.getAverage()).thenReturn(11203131L);
    when(summary.getStateSizeStats()).thenReturn(stateSizeSummary);
    when(summary.getEndToEndDurationStats()).thenReturn(durationSummary);
    when(summary.getAlignmentBufferedStats()).thenReturn(alignmentBufferedSummary);
    // Latest
    CompletedCheckpointStats latestCompleted = mock(CompletedCheckpointStats.class);
    when(latestCompleted.getCheckpointId()).thenReturn(1992139L);
    when(latestCompleted.getTriggerTimestamp()).thenReturn(1919191900L);
    when(latestCompleted.getLatestAckTimestamp()).thenReturn(1977791901L);
    when(latestCompleted.getStateSize()).thenReturn(111939272822L);
    when(latestCompleted.getEndToEndDuration()).thenReturn(121191L);
    when(latestCompleted.getAlignmentBuffered()).thenReturn(1L);
    when(latestCompleted.getExternalPath()).thenReturn("latest-completed-external-path");
    CompletedCheckpointStats latestSavepoint = mock(CompletedCheckpointStats.class);
    when(latestSavepoint.getCheckpointId()).thenReturn(1992140L);
    when(latestSavepoint.getTriggerTimestamp()).thenReturn(1919191900L);
    when(latestSavepoint.getLatestAckTimestamp()).thenReturn(1977791901L);
    when(latestSavepoint.getStateSize()).thenReturn(111939272822L);
    when(latestSavepoint.getEndToEndDuration()).thenReturn(121191L);
    when(latestCompleted.getAlignmentBuffered()).thenReturn(182813L);
    when(latestSavepoint.getExternalPath()).thenReturn("savepoint-external-path");
    FailedCheckpointStats latestFailed = mock(FailedCheckpointStats.class);
    when(latestFailed.getCheckpointId()).thenReturn(1112L);
    when(latestFailed.getTriggerTimestamp()).thenReturn(12828L);
    when(latestFailed.getLatestAckTimestamp()).thenReturn(1901L);
    when(latestFailed.getFailureTimestamp()).thenReturn(11999976L);
    when(latestFailed.getStateSize()).thenReturn(111L);
    when(latestFailed.getEndToEndDuration()).thenReturn(12L);
    when(latestFailed.getAlignmentBuffered()).thenReturn(2L);
    when(latestFailed.getFailureMessage()).thenReturn("expected cause");
    RestoredCheckpointStats latestRestored = mock(RestoredCheckpointStats.class);
    when(latestRestored.getCheckpointId()).thenReturn(1199L);
    when(latestRestored.getRestoreTimestamp()).thenReturn(434242L);
    when(latestRestored.getProperties()).thenReturn(CheckpointProperties.forStandardSavepoint());
    when(latestRestored.getExternalPath()).thenReturn("restored savepoint path");
    // History
    CheckpointStatsHistory history = mock(CheckpointStatsHistory.class);
    List<AbstractCheckpointStats> checkpoints = new ArrayList<>();
    PendingCheckpointStats inProgress = mock(PendingCheckpointStats.class);
    when(inProgress.getCheckpointId()).thenReturn(1992141L);
    when(inProgress.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
    when(inProgress.getProperties()).thenReturn(CheckpointProperties.forStandardCheckpoint());
    when(inProgress.getTriggerTimestamp()).thenReturn(1919191900L);
    when(inProgress.getLatestAckTimestamp()).thenReturn(1977791901L);
    when(inProgress.getStateSize()).thenReturn(111939272822L);
    when(inProgress.getEndToEndDuration()).thenReturn(121191L);
    when(inProgress.getAlignmentBuffered()).thenReturn(1L);
    when(inProgress.getNumberOfSubtasks()).thenReturn(501);
    when(inProgress.getNumberOfAcknowledgedSubtasks()).thenReturn(101);
    CompletedCheckpointStats completedSavepoint = mock(CompletedCheckpointStats.class);
    when(completedSavepoint.getCheckpointId()).thenReturn(1322139L);
    when(completedSavepoint.getStatus()).thenReturn(CheckpointStatsStatus.COMPLETED);
    when(completedSavepoint.getProperties()).thenReturn(CheckpointProperties.forStandardSavepoint());
    when(completedSavepoint.getTriggerTimestamp()).thenReturn(191900L);
    when(completedSavepoint.getLatestAckTimestamp()).thenReturn(197791901L);
    when(completedSavepoint.getStateSize()).thenReturn(1119822L);
    when(completedSavepoint.getEndToEndDuration()).thenReturn(12191L);
    when(completedSavepoint.getAlignmentBuffered()).thenReturn(111L);
    when(completedSavepoint.getNumberOfSubtasks()).thenReturn(33501);
    when(completedSavepoint.getNumberOfAcknowledgedSubtasks()).thenReturn(211);
    when(completedSavepoint.isDiscarded()).thenReturn(true);
    when(completedSavepoint.getExternalPath()).thenReturn("completed-external-path");
    FailedCheckpointStats failed = mock(FailedCheckpointStats.class);
    when(failed.getCheckpointId()).thenReturn(110719L);
    when(failed.getStatus()).thenReturn(CheckpointStatsStatus.FAILED);
    when(failed.getProperties()).thenReturn(CheckpointProperties.forStandardCheckpoint());
    when(failed.getTriggerTimestamp()).thenReturn(191900L);
    when(failed.getLatestAckTimestamp()).thenReturn(197791901L);
    when(failed.getStateSize()).thenReturn(1119822L);
    when(failed.getEndToEndDuration()).thenReturn(12191L);
    when(failed.getAlignmentBuffered()).thenReturn(111L);
    when(failed.getNumberOfSubtasks()).thenReturn(33501);
    when(failed.getNumberOfAcknowledgedSubtasks()).thenReturn(1);
    when(failed.getFailureTimestamp()).thenReturn(119230L);
    when(failed.getFailureMessage()).thenReturn("failure message");
    checkpoints.add(inProgress);
    checkpoints.add(completedSavepoint);
    checkpoints.add(failed);
    when(history.getCheckpoints()).thenReturn(checkpoints);
    when(history.getLatestCompletedCheckpoint()).thenReturn(latestCompleted);
    when(history.getLatestSavepoint()).thenReturn(latestSavepoint);
    when(history.getLatestFailedCheckpoint()).thenReturn(latestFailed);
    CheckpointStatsSnapshot snapshot = mock(CheckpointStatsSnapshot.class);
    when(snapshot.getCounts()).thenReturn(counts);
    when(snapshot.getSummaryStats()).thenReturn(summary);
    when(snapshot.getHistory()).thenReturn(history);
    when(snapshot.getLatestRestoredCheckpoint()).thenReturn(latestRestored);
    AccessExecutionGraph graph = mock(AccessExecutionGraph.class);
    when(graph.getCheckpointStatsSnapshot()).thenReturn(snapshot);
    return new TestCheckpointStats(graph, counts, stateSizeSummary, durationSummary, alignmentBufferedSummary, summary, latestCompleted, latestSavepoint, latestFailed, latestRestored, inProgress, completedSavepoint, failed, history, snapshot);
}

Example 5 with FailedCheckpointStats

use of org.apache.flink.runtime.checkpoint.FailedCheckpointStats in project flink by apache.

the class CheckpointStatsDetailsHandler method createCheckpointDetailsJson.

public static String createCheckpointDetailsJson(AbstractCheckpointStats checkpoint) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    gen.writeNumberField("id", checkpoint.getCheckpointId());
    gen.writeStringField("status", checkpoint.getStatus().toString());
    gen.writeBooleanField("is_savepoint", checkpoint.getProperties().isSavepoint());
    gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp());
    gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp());
    gen.writeNumberField("state_size", checkpoint.getStateSize());
    gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration());
    gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered());
    gen.writeNumberField("num_subtasks", checkpoint.getNumberOfSubtasks());
    gen.writeNumberField("num_acknowledged_subtasks", checkpoint.getNumberOfAcknowledgedSubtasks());
    if (checkpoint.getStatus().isCompleted()) {
        // --- Completed ---
        CompletedCheckpointStats completed = (CompletedCheckpointStats) checkpoint;
        String externalPath = completed.getExternalPath();
        if (externalPath != null) {
            gen.writeStringField("external_path", externalPath);
        }
        gen.writeBooleanField("discarded", completed.isDiscarded());
    } else if (checkpoint.getStatus().isFailed()) {
        // --- Failed ---
        FailedCheckpointStats failed = (FailedCheckpointStats) checkpoint;
        gen.writeNumberField("failure_timestamp", failed.getFailureTimestamp());
        String failureMsg = failed.getFailureMessage();
        if (failureMsg != null) {
            gen.writeStringField("failure_message", failureMsg);
        }
    }
    gen.writeObjectFieldStart("tasks");
    for (TaskStateStats taskStats : checkpoint.getAllTaskStateStats()) {
        gen.writeObjectFieldStart(taskStats.getJobVertexId().toString());
        gen.writeNumberField("latest_ack_timestamp", taskStats.getLatestAckTimestamp());
        gen.writeNumberField("state_size", taskStats.getStateSize());
        gen.writeNumberField("end_to_end_duration", taskStats.getEndToEndDuration(checkpoint.getTriggerTimestamp()));
        gen.writeNumberField("alignment_buffered", taskStats.getAlignmentBuffered());
        gen.writeNumberField("num_subtasks", taskStats.getNumberOfSubtasks());
        gen.writeNumberField("num_acknowledged_subtasks", taskStats.getNumberOfAcknowledgedSubtasks());
        gen.writeEndObject();
    }
    gen.writeEndObject();
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}