Examples with StatsSummarySnapshot org.apache.flink.runtime.checkpoint.StatsSummarySnapshot used on opensource projects

Example 1 with StatsSummarySnapshot

use of org.apache.flink.runtime.checkpoint.StatsSummarySnapshot in project flink by apache.

the class ArchivedExecutionGraphTest method compareExecutionGraph.

private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionGraph
    // -------------------------------------------------------------------------------------------------------------
    assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
    assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
    assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
    assertEquals(runtimeGraph.getState(), archivedGraph.getState());
    assertEquals(runtimeGraph.getFailureInfo().getExceptionAsString(), archivedGraph.getFailureInfo().getExceptionAsString());
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
    assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());
    // -------------------------------------------------------------------------------------------------------------
    // CheckpointStats
    // -------------------------------------------------------------------------------------------------------------
    CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
    CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();
    List<Function<CompletedCheckpointStatsSummarySnapshot, StatsSummarySnapshot>> meters = asList(CompletedCheckpointStatsSummarySnapshot::getEndToEndDurationStats, CompletedCheckpointStatsSummarySnapshot::getPersistedDataStats, CompletedCheckpointStatsSummarySnapshot::getProcessedDataStats, CompletedCheckpointStatsSummarySnapshot::getStateSizeStats);
    List<Function<StatsSummarySnapshot, Object>> aggs = asList(StatsSummarySnapshot::getAverage, StatsSummarySnapshot::getMinimum, StatsSummarySnapshot::getMaximum, StatsSummarySnapshot::getSum, StatsSummarySnapshot::getCount, s -> s.getQuantile(0.5d), s -> s.getQuantile(0.9d), s -> s.getQuantile(0.95d), s -> s.getQuantile(0.99d), s -> s.getQuantile(0.999d));
    for (Function<CompletedCheckpointStatsSummarySnapshot, StatsSummarySnapshot> meter : meters) {
        StatsSummarySnapshot runtime = meter.apply(runtimeSnapshot.getSummaryStats());
        StatsSummarySnapshot archived = meter.apply(runtimeSnapshot.getSummaryStats());
        for (Function<StatsSummarySnapshot, Object> agg : aggs) {
            assertEquals(agg.apply(runtime), agg.apply(archived));
        }
    }
    assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());
    // -------------------------------------------------------------------------------------------------------------
    // ArchivedExecutionConfig
    // -------------------------------------------------------------------------------------------------------------
    ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
    ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();
    assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
    assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
    assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
    assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
    assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
    assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));
    // -------------------------------------------------------------------------------------------------------------
    // StringifiedAccumulators
    // -------------------------------------------------------------------------------------------------------------
    compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
    compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());
    // -------------------------------------------------------------------------------------------------------------
    // JobVertices
    // -------------------------------------------------------------------------------------------------------------
    Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
    Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();
    for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
        compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
    }
    Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
    Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();
    while (runtimeTopologicalVertices.hasNext()) {
        assertTrue(archiveTopologicaldVertices.hasNext());
        compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
    }
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionVertices
    // -------------------------------------------------------------------------------------------------------------
    Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
    Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();
    while (runtimeExecutionVertices.hasNext()) {
        assertTrue(archivedExecutionVertices.hasNext());
        compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
    }
}