use of org.apache.flink.runtime.checkpoint.PendingCheckpointStats in project flink by apache.
the class CheckpointStatsDetailsHandlerTest method testCheckpointDetailsRequestInProgressCheckpoint.
/**
* Tests a checkpoint details request for an in progress checkpoint.
*/
@Test
public void testCheckpointDetailsRequestInProgressCheckpoint() throws Exception {
PendingCheckpointStats checkpoint = mock(PendingCheckpointStats.class);
when(checkpoint.getCheckpointId()).thenReturn(1992139L);
when(checkpoint.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
when(checkpoint.getProperties()).thenReturn(CheckpointProperties.forStandardCheckpoint());
when(checkpoint.getTriggerTimestamp()).thenReturn(1919191900L);
when(checkpoint.getLatestAckTimestamp()).thenReturn(1977791901L);
when(checkpoint.getStateSize()).thenReturn(111939272822L);
when(checkpoint.getEndToEndDuration()).thenReturn(121191L);
when(checkpoint.getAlignmentBuffered()).thenReturn(1L);
when(checkpoint.getNumberOfSubtasks()).thenReturn(501);
when(checkpoint.getNumberOfAcknowledgedSubtasks()).thenReturn(101);
List<TaskStateStats> taskStats = new ArrayList<>();
TaskStateStats task1 = createTaskStateStats();
TaskStateStats task2 = createTaskStateStats();
taskStats.add(task1);
taskStats.add(task2);
when(checkpoint.getAllTaskStateStats()).thenReturn(taskStats);
JsonNode rootNode = triggerRequest(checkpoint);
assertEquals(checkpoint.getCheckpointId(), rootNode.get("id").asLong());
assertEquals(checkpoint.getStatus().toString(), rootNode.get("status").asText());
assertEquals(checkpoint.getProperties().isSavepoint(), rootNode.get("is_savepoint").asBoolean());
assertEquals(checkpoint.getTriggerTimestamp(), rootNode.get("trigger_timestamp").asLong());
assertEquals(checkpoint.getLatestAckTimestamp(), rootNode.get("latest_ack_timestamp").asLong());
assertEquals(checkpoint.getStateSize(), rootNode.get("state_size").asLong());
assertEquals(checkpoint.getEndToEndDuration(), rootNode.get("end_to_end_duration").asLong());
assertEquals(checkpoint.getAlignmentBuffered(), rootNode.get("alignment_buffered").asLong());
assertEquals(checkpoint.getNumberOfSubtasks(), rootNode.get("num_subtasks").asInt());
assertEquals(checkpoint.getNumberOfAcknowledgedSubtasks(), rootNode.get("num_acknowledged_subtasks").asInt());
verifyTaskNodes(taskStats, rootNode);
}
use of org.apache.flink.runtime.checkpoint.PendingCheckpointStats in project flink by apache.
the class CheckpointStatsHandlerTest method compareCheckpointStats.
private static void compareCheckpointStats(TestCheckpointStats checkpointStats, JsonNode rootNode) {
CheckpointStatsCounts counts = checkpointStats.counts;
JsonNode countNode = rootNode.get("counts");
assertEquals(counts.getNumberOfRestoredCheckpoints(), countNode.get("restored").asLong());
assertEquals(counts.getTotalNumberOfCheckpoints(), countNode.get("total").asLong());
assertEquals(counts.getNumberOfInProgressCheckpoints(), countNode.get("in_progress").asLong());
assertEquals(counts.getNumberOfCompletedCheckpoints(), countNode.get("completed").asLong());
assertEquals(counts.getNumberOfFailedCheckpoints(), countNode.get("failed").asLong());
MinMaxAvgStats stateSizeSummary = checkpointStats.stateSizeSummary;
JsonNode summaryNode = rootNode.get("summary");
JsonNode sizeSummaryNode = summaryNode.get("state_size");
assertEquals(stateSizeSummary.getMinimum(), sizeSummaryNode.get("min").asLong());
assertEquals(stateSizeSummary.getMaximum(), sizeSummaryNode.get("max").asLong());
assertEquals(stateSizeSummary.getAverage(), sizeSummaryNode.get("avg").asLong());
MinMaxAvgStats durationSummary = checkpointStats.durationSummary;
JsonNode durationSummaryNode = summaryNode.get("end_to_end_duration");
assertEquals(durationSummary.getMinimum(), durationSummaryNode.get("min").asLong());
assertEquals(durationSummary.getMaximum(), durationSummaryNode.get("max").asLong());
assertEquals(durationSummary.getAverage(), durationSummaryNode.get("avg").asLong());
MinMaxAvgStats alignmentBufferedSummary = checkpointStats.alignmentBufferedSummary;
JsonNode alignmentBufferedNode = summaryNode.get("alignment_buffered");
assertEquals(alignmentBufferedSummary.getMinimum(), alignmentBufferedNode.get("min").asLong());
assertEquals(alignmentBufferedSummary.getMaximum(), alignmentBufferedNode.get("max").asLong());
assertEquals(alignmentBufferedSummary.getAverage(), alignmentBufferedNode.get("avg").asLong());
CompletedCheckpointStats latestCompleted = checkpointStats.latestCompleted;
JsonNode latestNode = rootNode.get("latest");
JsonNode latestCheckpointNode = latestNode.get("completed");
assertEquals(latestCompleted.getCheckpointId(), latestCheckpointNode.get("id").asLong());
assertEquals(latestCompleted.getTriggerTimestamp(), latestCheckpointNode.get("trigger_timestamp").asLong());
assertEquals(latestCompleted.getLatestAckTimestamp(), latestCheckpointNode.get("latest_ack_timestamp").asLong());
assertEquals(latestCompleted.getStateSize(), latestCheckpointNode.get("state_size").asLong());
assertEquals(latestCompleted.getEndToEndDuration(), latestCheckpointNode.get("end_to_end_duration").asLong());
assertEquals(latestCompleted.getAlignmentBuffered(), latestCheckpointNode.get("alignment_buffered").asLong());
assertEquals(latestCompleted.getExternalPath(), latestCheckpointNode.get("external_path").asText());
CompletedCheckpointStats latestSavepoint = checkpointStats.latestSavepoint;
JsonNode latestSavepointNode = latestNode.get("savepoint");
assertEquals(latestSavepoint.getCheckpointId(), latestSavepointNode.get("id").asLong());
assertEquals(latestSavepoint.getTriggerTimestamp(), latestSavepointNode.get("trigger_timestamp").asLong());
assertEquals(latestSavepoint.getLatestAckTimestamp(), latestSavepointNode.get("latest_ack_timestamp").asLong());
assertEquals(latestSavepoint.getStateSize(), latestSavepointNode.get("state_size").asLong());
assertEquals(latestSavepoint.getEndToEndDuration(), latestSavepointNode.get("end_to_end_duration").asLong());
assertEquals(latestSavepoint.getAlignmentBuffered(), latestSavepointNode.get("alignment_buffered").asLong());
assertEquals(latestSavepoint.getExternalPath(), latestSavepointNode.get("external_path").asText());
FailedCheckpointStats latestFailed = checkpointStats.latestFailed;
JsonNode latestFailedNode = latestNode.get("failed");
assertEquals(latestFailed.getCheckpointId(), latestFailedNode.get("id").asLong());
assertEquals(latestFailed.getTriggerTimestamp(), latestFailedNode.get("trigger_timestamp").asLong());
assertEquals(latestFailed.getLatestAckTimestamp(), latestFailedNode.get("latest_ack_timestamp").asLong());
assertEquals(latestFailed.getStateSize(), latestFailedNode.get("state_size").asLong());
assertEquals(latestFailed.getEndToEndDuration(), latestFailedNode.get("end_to_end_duration").asLong());
assertEquals(latestFailed.getAlignmentBuffered(), latestFailedNode.get("alignment_buffered").asLong());
assertEquals(latestFailed.getFailureTimestamp(), latestFailedNode.get("failure_timestamp").asLong());
assertEquals(latestFailed.getFailureMessage(), latestFailedNode.get("failure_message").asText());
RestoredCheckpointStats latestRestored = checkpointStats.latestRestored;
JsonNode latestRestoredNode = latestNode.get("restored");
assertEquals(latestRestored.getCheckpointId(), latestRestoredNode.get("id").asLong());
assertEquals(latestRestored.getRestoreTimestamp(), latestRestoredNode.get("restore_timestamp").asLong());
assertEquals(latestRestored.getProperties().isSavepoint(), latestRestoredNode.get("is_savepoint").asBoolean());
assertEquals(latestRestored.getExternalPath(), latestRestoredNode.get("external_path").asText());
JsonNode historyNode = rootNode.get("history");
Iterator<JsonNode> it = historyNode.iterator();
assertTrue(it.hasNext());
JsonNode inProgressNode = it.next();
PendingCheckpointStats inProgress = checkpointStats.inProgress;
compareInProgressCheckpoint(inProgress, inProgressNode);
assertTrue(it.hasNext());
JsonNode completedSavepointNode = it.next();
CompletedCheckpointStats completedSavepoint = checkpointStats.completedSavepoint;
compareCompletedSavepoint(completedSavepoint, completedSavepointNode);
assertTrue(it.hasNext());
JsonNode failedNode = it.next();
FailedCheckpointStats failed = checkpointStats.failed;
compareFailedCheckpoint(failed, failedNode);
assertFalse(it.hasNext());
}
use of org.apache.flink.runtime.checkpoint.PendingCheckpointStats in project flink by apache.
the class CheckpointStatsHandlerTest method createTestCheckpointStats.
private static TestCheckpointStats createTestCheckpointStats() {
// Counts
CheckpointStatsCounts counts = mock(CheckpointStatsCounts.class);
when(counts.getNumberOfRestoredCheckpoints()).thenReturn(123123123L);
when(counts.getTotalNumberOfCheckpoints()).thenReturn(12981231203L);
when(counts.getNumberOfInProgressCheckpoints()).thenReturn(191919);
when(counts.getNumberOfCompletedCheckpoints()).thenReturn(882828200L);
when(counts.getNumberOfFailedCheckpoints()).thenReturn(99171510L);
// Summary
CompletedCheckpointStatsSummary summary = mock(CompletedCheckpointStatsSummary.class);
MinMaxAvgStats stateSizeSummary = mock(MinMaxAvgStats.class);
when(stateSizeSummary.getMinimum()).thenReturn(81238123L);
when(stateSizeSummary.getMaximum()).thenReturn(19919191999L);
when(stateSizeSummary.getAverage()).thenReturn(1133L);
MinMaxAvgStats durationSummary = mock(MinMaxAvgStats.class);
when(durationSummary.getMinimum()).thenReturn(1182L);
when(durationSummary.getMaximum()).thenReturn(88654L);
when(durationSummary.getAverage()).thenReturn(171L);
MinMaxAvgStats alignmentBufferedSummary = mock(MinMaxAvgStats.class);
when(alignmentBufferedSummary.getMinimum()).thenReturn(81818181899L);
when(alignmentBufferedSummary.getMaximum()).thenReturn(89999911118654L);
when(alignmentBufferedSummary.getAverage()).thenReturn(11203131L);
when(summary.getStateSizeStats()).thenReturn(stateSizeSummary);
when(summary.getEndToEndDurationStats()).thenReturn(durationSummary);
when(summary.getAlignmentBufferedStats()).thenReturn(alignmentBufferedSummary);
// Latest
CompletedCheckpointStats latestCompleted = mock(CompletedCheckpointStats.class);
when(latestCompleted.getCheckpointId()).thenReturn(1992139L);
when(latestCompleted.getTriggerTimestamp()).thenReturn(1919191900L);
when(latestCompleted.getLatestAckTimestamp()).thenReturn(1977791901L);
when(latestCompleted.getStateSize()).thenReturn(111939272822L);
when(latestCompleted.getEndToEndDuration()).thenReturn(121191L);
when(latestCompleted.getAlignmentBuffered()).thenReturn(1L);
when(latestCompleted.getExternalPath()).thenReturn("latest-completed-external-path");
CompletedCheckpointStats latestSavepoint = mock(CompletedCheckpointStats.class);
when(latestSavepoint.getCheckpointId()).thenReturn(1992140L);
when(latestSavepoint.getTriggerTimestamp()).thenReturn(1919191900L);
when(latestSavepoint.getLatestAckTimestamp()).thenReturn(1977791901L);
when(latestSavepoint.getStateSize()).thenReturn(111939272822L);
when(latestSavepoint.getEndToEndDuration()).thenReturn(121191L);
when(latestCompleted.getAlignmentBuffered()).thenReturn(182813L);
when(latestSavepoint.getExternalPath()).thenReturn("savepoint-external-path");
FailedCheckpointStats latestFailed = mock(FailedCheckpointStats.class);
when(latestFailed.getCheckpointId()).thenReturn(1112L);
when(latestFailed.getTriggerTimestamp()).thenReturn(12828L);
when(latestFailed.getLatestAckTimestamp()).thenReturn(1901L);
when(latestFailed.getFailureTimestamp()).thenReturn(11999976L);
when(latestFailed.getStateSize()).thenReturn(111L);
when(latestFailed.getEndToEndDuration()).thenReturn(12L);
when(latestFailed.getAlignmentBuffered()).thenReturn(2L);
when(latestFailed.getFailureMessage()).thenReturn("expected cause");
RestoredCheckpointStats latestRestored = mock(RestoredCheckpointStats.class);
when(latestRestored.getCheckpointId()).thenReturn(1199L);
when(latestRestored.getRestoreTimestamp()).thenReturn(434242L);
when(latestRestored.getProperties()).thenReturn(CheckpointProperties.forStandardSavepoint());
when(latestRestored.getExternalPath()).thenReturn("restored savepoint path");
// History
CheckpointStatsHistory history = mock(CheckpointStatsHistory.class);
List<AbstractCheckpointStats> checkpoints = new ArrayList<>();
PendingCheckpointStats inProgress = mock(PendingCheckpointStats.class);
when(inProgress.getCheckpointId()).thenReturn(1992141L);
when(inProgress.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
when(inProgress.getProperties()).thenReturn(CheckpointProperties.forStandardCheckpoint());
when(inProgress.getTriggerTimestamp()).thenReturn(1919191900L);
when(inProgress.getLatestAckTimestamp()).thenReturn(1977791901L);
when(inProgress.getStateSize()).thenReturn(111939272822L);
when(inProgress.getEndToEndDuration()).thenReturn(121191L);
when(inProgress.getAlignmentBuffered()).thenReturn(1L);
when(inProgress.getNumberOfSubtasks()).thenReturn(501);
when(inProgress.getNumberOfAcknowledgedSubtasks()).thenReturn(101);
CompletedCheckpointStats completedSavepoint = mock(CompletedCheckpointStats.class);
when(completedSavepoint.getCheckpointId()).thenReturn(1322139L);
when(completedSavepoint.getStatus()).thenReturn(CheckpointStatsStatus.COMPLETED);
when(completedSavepoint.getProperties()).thenReturn(CheckpointProperties.forStandardSavepoint());
when(completedSavepoint.getTriggerTimestamp()).thenReturn(191900L);
when(completedSavepoint.getLatestAckTimestamp()).thenReturn(197791901L);
when(completedSavepoint.getStateSize()).thenReturn(1119822L);
when(completedSavepoint.getEndToEndDuration()).thenReturn(12191L);
when(completedSavepoint.getAlignmentBuffered()).thenReturn(111L);
when(completedSavepoint.getNumberOfSubtasks()).thenReturn(33501);
when(completedSavepoint.getNumberOfAcknowledgedSubtasks()).thenReturn(211);
when(completedSavepoint.isDiscarded()).thenReturn(true);
when(completedSavepoint.getExternalPath()).thenReturn("completed-external-path");
FailedCheckpointStats failed = mock(FailedCheckpointStats.class);
when(failed.getCheckpointId()).thenReturn(110719L);
when(failed.getStatus()).thenReturn(CheckpointStatsStatus.FAILED);
when(failed.getProperties()).thenReturn(CheckpointProperties.forStandardCheckpoint());
when(failed.getTriggerTimestamp()).thenReturn(191900L);
when(failed.getLatestAckTimestamp()).thenReturn(197791901L);
when(failed.getStateSize()).thenReturn(1119822L);
when(failed.getEndToEndDuration()).thenReturn(12191L);
when(failed.getAlignmentBuffered()).thenReturn(111L);
when(failed.getNumberOfSubtasks()).thenReturn(33501);
when(failed.getNumberOfAcknowledgedSubtasks()).thenReturn(1);
when(failed.getFailureTimestamp()).thenReturn(119230L);
when(failed.getFailureMessage()).thenReturn("failure message");
checkpoints.add(inProgress);
checkpoints.add(completedSavepoint);
checkpoints.add(failed);
when(history.getCheckpoints()).thenReturn(checkpoints);
when(history.getLatestCompletedCheckpoint()).thenReturn(latestCompleted);
when(history.getLatestSavepoint()).thenReturn(latestSavepoint);
when(history.getLatestFailedCheckpoint()).thenReturn(latestFailed);
CheckpointStatsSnapshot snapshot = mock(CheckpointStatsSnapshot.class);
when(snapshot.getCounts()).thenReturn(counts);
when(snapshot.getSummaryStats()).thenReturn(summary);
when(snapshot.getHistory()).thenReturn(history);
when(snapshot.getLatestRestoredCheckpoint()).thenReturn(latestRestored);
AccessExecutionGraph graph = mock(AccessExecutionGraph.class);
when(graph.getCheckpointStatsSnapshot()).thenReturn(snapshot);
return new TestCheckpointStats(graph, counts, stateSizeSummary, durationSummary, alignmentBufferedSummary, summary, latestCompleted, latestSavepoint, latestFailed, latestRestored, inProgress, completedSavepoint, failed, history, snapshot);
}
use of org.apache.flink.runtime.checkpoint.PendingCheckpointStats in project flink by apache.
the class CheckpointStatsSubtaskDetailsHandlerTest method testSubtaskRequest.
/**
* Tests a subtask details request.
*/
@Test
public void testSubtaskRequest() throws Exception {
PendingCheckpointStats checkpoint = mock(PendingCheckpointStats.class);
when(checkpoint.getCheckpointId()).thenReturn(1992139L);
when(checkpoint.getStatus()).thenReturn(CheckpointStatsStatus.IN_PROGRESS);
// ack timestamp = duration
when(checkpoint.getTriggerTimestamp()).thenReturn(0L);
TaskStateStats task = createTaskStateStats(1237);
when(checkpoint.getTaskStateStats(any(JobVertexID.class))).thenReturn(task);
JsonNode rootNode = triggerRequest(checkpoint);
assertEquals(checkpoint.getCheckpointId(), rootNode.get("id").asLong());
assertEquals(checkpoint.getStatus().toString(), rootNode.get("status").asText());
verifyTaskNode(rootNode, task, checkpoint.getTriggerTimestamp());
}
use of org.apache.flink.runtime.checkpoint.PendingCheckpointStats in project flink by apache.
the class CheckpointStatsSubtaskDetailsHandlerTest method testJobVertexNotFound.
/**
* Test lookup of not existing job vertex ID in checkpoint.
*/
@Test
public void testJobVertexNotFound() throws Exception {
PendingCheckpointStats inProgress = mock(PendingCheckpointStats.class);
// not found
when(inProgress.getTaskStateStats(any(JobVertexID.class))).thenReturn(null);
CheckpointStatsHistory history = mock(CheckpointStatsHistory.class);
when(history.getCheckpointById(anyLong())).thenReturn(inProgress);
CheckpointStatsSnapshot snapshot = mock(CheckpointStatsSnapshot.class);
when(snapshot.getHistory()).thenReturn(history);
AccessExecutionGraph graph = mock(AccessExecutionGraph.class);
when(graph.getCheckpointStatsSnapshot()).thenReturn(snapshot);
CheckpointStatsDetailsSubtasksHandler handler = new CheckpointStatsDetailsSubtasksHandler(mock(ExecutionGraphHolder.class), new CheckpointStatsCache(0));
Map<String, String> params = new HashMap<>();
params.put("checkpointid", "123");
params.put("vertexid", new JobVertexID().toString());
String json = handler.handleRequest(graph, params);
assertEquals("{}", json);
verify(inProgress, times(1)).getTaskStateStats(any(JobVertexID.class));
}
Aggregations