Examples with CompletedCheckpointStatsSummarySnapshot org.apache.flink.runtime.checkpoint.CompletedCheckpointStatsSummarySnapshot used on opensource projects

Example 1 with CompletedCheckpointStatsSummarySnapshot

use of org.apache.flink.runtime.checkpoint.CompletedCheckpointStatsSummarySnapshot in project flink by apache.

the class CheckpointingStatisticsHandler method createCheckpointingStatistics.

private static CheckpointingStatistics createCheckpointingStatistics(AccessExecutionGraph executionGraph) throws RestHandlerException {
    final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot();
    if (checkpointStatsSnapshot == null) {
        throw new RestHandlerException("Checkpointing has not been enabled.", HttpResponseStatus.NOT_FOUND, RestHandlerException.LoggingBehavior.IGNORE);
    } else {
        final CheckpointStatsCounts checkpointStatsCounts = checkpointStatsSnapshot.getCounts();
        final CheckpointingStatistics.Counts counts = new CheckpointingStatistics.Counts(checkpointStatsCounts.getNumberOfRestoredCheckpoints(), checkpointStatsCounts.getTotalNumberOfCheckpoints(), checkpointStatsCounts.getNumberOfInProgressCheckpoints(), checkpointStatsCounts.getNumberOfCompletedCheckpoints(), checkpointStatsCounts.getNumberOfFailedCheckpoints());
        final CompletedCheckpointStatsSummarySnapshot checkpointStatsSummary = checkpointStatsSnapshot.getSummaryStats();
        final CheckpointingStatistics.Summary summary = new CheckpointingStatistics.Summary(StatsSummaryDto.valueOf(checkpointStatsSummary.getCheckpointedSize()), StatsSummaryDto.valueOf(checkpointStatsSummary.getStateSizeStats()), StatsSummaryDto.valueOf(checkpointStatsSummary.getEndToEndDurationStats()), new StatsSummaryDto(0, 0, 0, 0, 0, 0, 0, 0), StatsSummaryDto.valueOf(checkpointStatsSummary.getProcessedDataStats()), StatsSummaryDto.valueOf(checkpointStatsSummary.getPersistedDataStats()));
        final CheckpointStatsHistory checkpointStatsHistory = checkpointStatsSnapshot.getHistory();
        final CheckpointStatistics.CompletedCheckpointStatistics completed = checkpointStatsHistory.getLatestCompletedCheckpoint() != null ? (CheckpointStatistics.CompletedCheckpointStatistics) CheckpointStatistics.generateCheckpointStatistics(checkpointStatsHistory.getLatestCompletedCheckpoint(), false) : null;
        final CheckpointStatistics.CompletedCheckpointStatistics savepoint = checkpointStatsHistory.getLatestSavepoint() != null ? (CheckpointStatistics.CompletedCheckpointStatistics) CheckpointStatistics.generateCheckpointStatistics(checkpointStatsHistory.getLatestSavepoint(), false) : null;
        final CheckpointStatistics.FailedCheckpointStatistics failed = checkpointStatsHistory.getLatestFailedCheckpoint() != null ? (CheckpointStatistics.FailedCheckpointStatistics) CheckpointStatistics.generateCheckpointStatistics(checkpointStatsHistory.getLatestFailedCheckpoint(), false) : null;
        final RestoredCheckpointStats restoredCheckpointStats = checkpointStatsSnapshot.getLatestRestoredCheckpoint();
        final CheckpointingStatistics.RestoredCheckpointStatistics restored;
        if (restoredCheckpointStats == null) {
            restored = null;
        } else {
            restored = new CheckpointingStatistics.RestoredCheckpointStatistics(restoredCheckpointStats.getCheckpointId(), restoredCheckpointStats.getRestoreTimestamp(), restoredCheckpointStats.getProperties().isSavepoint(), restoredCheckpointStats.getExternalPath());
        }
        final CheckpointingStatistics.LatestCheckpoints latestCheckpoints = new CheckpointingStatistics.LatestCheckpoints(completed, savepoint, failed, restored);
        final List<CheckpointStatistics> history = new ArrayList<>(16);
        for (AbstractCheckpointStats abstractCheckpointStats : checkpointStatsSnapshot.getHistory().getCheckpoints()) {
            history.add(CheckpointStatistics.generateCheckpointStatistics(abstractCheckpointStats, false));
        }
        return new CheckpointingStatistics(counts, summary, latestCheckpoints, history);
    }
}

Example 2 with CompletedCheckpointStatsSummarySnapshot

use of org.apache.flink.runtime.checkpoint.CompletedCheckpointStatsSummarySnapshot in project flink by apache.

the class ArchivedExecutionGraphTest method compareExecutionGraph.

private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionGraph
    // -------------------------------------------------------------------------------------------------------------
    assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
    assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
    assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
    assertEquals(runtimeGraph.getState(), archivedGraph.getState());
    assertEquals(runtimeGraph.getFailureInfo().getExceptionAsString(), archivedGraph.getFailureInfo().getExceptionAsString());
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
    assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());
    // -------------------------------------------------------------------------------------------------------------
    // CheckpointStats
    // -------------------------------------------------------------------------------------------------------------
    CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
    CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();
    List<Function<CompletedCheckpointStatsSummarySnapshot, StatsSummarySnapshot>> meters = asList(CompletedCheckpointStatsSummarySnapshot::getEndToEndDurationStats, CompletedCheckpointStatsSummarySnapshot::getPersistedDataStats, CompletedCheckpointStatsSummarySnapshot::getProcessedDataStats, CompletedCheckpointStatsSummarySnapshot::getStateSizeStats);
    List<Function<StatsSummarySnapshot, Object>> aggs = asList(StatsSummarySnapshot::getAverage, StatsSummarySnapshot::getMinimum, StatsSummarySnapshot::getMaximum, StatsSummarySnapshot::getSum, StatsSummarySnapshot::getCount, s -> s.getQuantile(0.5d), s -> s.getQuantile(0.9d), s -> s.getQuantile(0.95d), s -> s.getQuantile(0.99d), s -> s.getQuantile(0.999d));
    for (Function<CompletedCheckpointStatsSummarySnapshot, StatsSummarySnapshot> meter : meters) {
        StatsSummarySnapshot runtime = meter.apply(runtimeSnapshot.getSummaryStats());
        StatsSummarySnapshot archived = meter.apply(runtimeSnapshot.getSummaryStats());
        for (Function<StatsSummarySnapshot, Object> agg : aggs) {
            assertEquals(agg.apply(runtime), agg.apply(archived));
        }
    }
    assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());
    // -------------------------------------------------------------------------------------------------------------
    // ArchivedExecutionConfig
    // -------------------------------------------------------------------------------------------------------------
    ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
    ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();
    assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
    assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
    assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
    assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
    assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
    assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));
    // -------------------------------------------------------------------------------------------------------------
    // StringifiedAccumulators
    // -------------------------------------------------------------------------------------------------------------
    compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
    compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());
    // -------------------------------------------------------------------------------------------------------------
    // JobVertices
    // -------------------------------------------------------------------------------------------------------------
    Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
    Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();
    for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
        compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
    }
    Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
    Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();
    while (runtimeTopologicalVertices.hasNext()) {
        assertTrue(archiveTopologicaldVertices.hasNext());
        compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
    }
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionVertices
    // -------------------------------------------------------------------------------------------------------------
    Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
    Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();
    while (runtimeExecutionVertices.hasNext()) {
        assertTrue(archivedExecutionVertices.hasNext());
        compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
    }
}