Examples with ArchivedExecutionConfig org.apache.flink.api.common.ArchivedExecutionConfig used on opensource projects

Example 1 with ArchivedExecutionConfig

use of org.apache.flink.api.common.ArchivedExecutionConfig in project flink by apache.

the class JobConfigHandler method createJobConfigJson.

public static String createJobConfigJson(AccessExecutionGraph graph) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    gen.writeStartObject();
    gen.writeStringField("jid", graph.getJobID().toString());
    gen.writeStringField("name", graph.getJobName());
    final ArchivedExecutionConfig summary = graph.getArchivedExecutionConfig();
    if (summary != null) {
        gen.writeObjectFieldStart("execution-config");
        gen.writeStringField("execution-mode", summary.getExecutionMode());
        gen.writeStringField("restart-strategy", summary.getRestartStrategyDescription());
        gen.writeNumberField("job-parallelism", summary.getParallelism());
        gen.writeBooleanField("object-reuse-mode", summary.getObjectReuseEnabled());
        Map<String, String> ucVals = summary.getGlobalJobParameters();
        if (ucVals != null) {
            gen.writeObjectFieldStart("user-config");
            for (Map.Entry<String, String> ucVal : ucVals.entrySet()) {
                gen.writeStringField(ucVal.getKey(), ucVal.getValue());
            }
            gen.writeEndObject();
        }
        gen.writeEndObject();
    }
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 2 with ArchivedExecutionConfig

use of org.apache.flink.api.common.ArchivedExecutionConfig in project flink by apache.

the class JobConfigHandlerTest method compareJobConfig.

private static void compareJobConfig(AccessExecutionGraph originalJob, String answer) throws IOException {
    JsonNode job = ArchivedJobGenerationUtils.mapper.readTree(answer);
    Assert.assertEquals(originalJob.getJobID().toString(), job.get("jid").asText());
    Assert.assertEquals(originalJob.getJobName(), job.get("name").asText());
    ArchivedExecutionConfig originalConfig = originalJob.getArchivedExecutionConfig();
    JsonNode config = job.get("execution-config");
    Assert.assertEquals(originalConfig.getExecutionMode(), config.get("execution-mode").asText());
    Assert.assertEquals(originalConfig.getRestartStrategyDescription(), config.get("restart-strategy").asText());
    Assert.assertEquals(originalConfig.getParallelism(), config.get("job-parallelism").asInt());
    Assert.assertEquals(originalConfig.getObjectReuseEnabled(), config.get("object-reuse-mode").asBoolean());
    Map<String, String> originalUserConfig = originalConfig.getGlobalJobParameters();
    JsonNode userConfig = config.get("user-config");
    for (Map.Entry<String, String> originalEntry : originalUserConfig.entrySet()) {
        Assert.assertEquals(originalEntry.getValue(), userConfig.get(originalEntry.getKey()).asText());
    }
}

Example 3 with ArchivedExecutionConfig

use of org.apache.flink.api.common.ArchivedExecutionConfig in project flink by apache.

the class ArchivedExecutionGraphTest method compareExecutionGraph.

private static void compareExecutionGraph(AccessExecutionGraph runtimeGraph, AccessExecutionGraph archivedGraph) throws IOException, ClassNotFoundException {
    assertTrue(archivedGraph.isArchived());
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionGraph
    // -------------------------------------------------------------------------------------------------------------
    assertEquals(runtimeGraph.getJsonPlan(), archivedGraph.getJsonPlan());
    assertEquals(runtimeGraph.getJobID(), archivedGraph.getJobID());
    assertEquals(runtimeGraph.getJobName(), archivedGraph.getJobName());
    assertEquals(runtimeGraph.getState(), archivedGraph.getState());
    assertEquals(runtimeGraph.getFailureCauseAsString(), archivedGraph.getFailureCauseAsString());
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CREATED), archivedGraph.getStatusTimestamp(JobStatus.CREATED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RUNNING), archivedGraph.getStatusTimestamp(JobStatus.RUNNING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILING), archivedGraph.getStatusTimestamp(JobStatus.FAILING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FAILED), archivedGraph.getStatusTimestamp(JobStatus.FAILED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELLING), archivedGraph.getStatusTimestamp(JobStatus.CANCELLING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.CANCELED), archivedGraph.getStatusTimestamp(JobStatus.CANCELED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.FINISHED), archivedGraph.getStatusTimestamp(JobStatus.FINISHED));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.RESTARTING), archivedGraph.getStatusTimestamp(JobStatus.RESTARTING));
    assertEquals(runtimeGraph.getStatusTimestamp(JobStatus.SUSPENDED), archivedGraph.getStatusTimestamp(JobStatus.SUSPENDED));
    assertEquals(runtimeGraph.isStoppable(), archivedGraph.isStoppable());
    // -------------------------------------------------------------------------------------------------------------
    // CheckpointStats
    // -------------------------------------------------------------------------------------------------------------
    CheckpointStatsSnapshot runtimeSnapshot = runtimeGraph.getCheckpointStatsSnapshot();
    CheckpointStatsSnapshot archivedSnapshot = archivedGraph.getCheckpointStatsSnapshot();
    assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getAverage());
    assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMinimum());
    assertEquals(runtimeSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum(), archivedSnapshot.getSummaryStats().getEndToEndDurationStats().getMaximum());
    assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getAverage(), archivedSnapshot.getSummaryStats().getStateSizeStats().getAverage());
    assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMinimum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMinimum());
    assertEquals(runtimeSnapshot.getSummaryStats().getStateSizeStats().getMaximum(), archivedSnapshot.getSummaryStats().getStateSizeStats().getMaximum());
    assertEquals(runtimeSnapshot.getCounts().getTotalNumberOfCheckpoints(), archivedSnapshot.getCounts().getTotalNumberOfCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfCompletedCheckpoints(), archivedSnapshot.getCounts().getNumberOfCompletedCheckpoints());
    assertEquals(runtimeSnapshot.getCounts().getNumberOfInProgressCheckpoints(), archivedSnapshot.getCounts().getNumberOfInProgressCheckpoints());
    // -------------------------------------------------------------------------------------------------------------
    // ArchivedExecutionConfig
    // -------------------------------------------------------------------------------------------------------------
    ArchivedExecutionConfig runtimeConfig = runtimeGraph.getArchivedExecutionConfig();
    ArchivedExecutionConfig archivedConfig = archivedGraph.getArchivedExecutionConfig();
    assertEquals(runtimeConfig.getExecutionMode(), archivedConfig.getExecutionMode());
    assertEquals(runtimeConfig.getParallelism(), archivedConfig.getParallelism());
    assertEquals(runtimeConfig.getObjectReuseEnabled(), archivedConfig.getObjectReuseEnabled());
    assertEquals(runtimeConfig.getRestartStrategyDescription(), archivedConfig.getRestartStrategyDescription());
    assertNotNull(archivedConfig.getGlobalJobParameters().get("hello"));
    assertEquals(runtimeConfig.getGlobalJobParameters().get("hello"), archivedConfig.getGlobalJobParameters().get("hello"));
    // -------------------------------------------------------------------------------------------------------------
    // StringifiedAccumulators
    // -------------------------------------------------------------------------------------------------------------
    compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
    compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());
    // -------------------------------------------------------------------------------------------------------------
    // JobVertices
    // -------------------------------------------------------------------------------------------------------------
    Map<JobVertexID, ? extends AccessExecutionJobVertex> runtimeVertices = runtimeGraph.getAllVertices();
    Map<JobVertexID, ? extends AccessExecutionJobVertex> archivedVertices = archivedGraph.getAllVertices();
    for (Map.Entry<JobVertexID, ? extends AccessExecutionJobVertex> vertex : runtimeVertices.entrySet()) {
        compareExecutionJobVertex(vertex.getValue(), archivedVertices.get(vertex.getKey()));
    }
    Iterator<? extends AccessExecutionJobVertex> runtimeTopologicalVertices = runtimeGraph.getVerticesTopologically().iterator();
    Iterator<? extends AccessExecutionJobVertex> archiveTopologicaldVertices = archivedGraph.getVerticesTopologically().iterator();
    while (runtimeTopologicalVertices.hasNext()) {
        assertTrue(archiveTopologicaldVertices.hasNext());
        compareExecutionJobVertex(runtimeTopologicalVertices.next(), archiveTopologicaldVertices.next());
    }
    // -------------------------------------------------------------------------------------------------------------
    // ExecutionVertices
    // -------------------------------------------------------------------------------------------------------------
    Iterator<? extends AccessExecutionVertex> runtimeExecutionVertices = runtimeGraph.getAllExecutionVertices().iterator();
    Iterator<? extends AccessExecutionVertex> archivedExecutionVertices = archivedGraph.getAllExecutionVertices().iterator();
    while (runtimeExecutionVertices.hasNext()) {
        assertTrue(archivedExecutionVertices.hasNext());
        compareExecutionVertex(runtimeExecutionVertices.next(), archivedExecutionVertices.next());
    }
}