Examples with JobSnapshottingSettings org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings used on opensource projects

Example 6 with JobSnapshottingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings in project flink by apache.

the class CheckpointConfigHandler method createCheckpointConfigJson.

private static String createCheckpointConfigJson(AccessExecutionGraph graph) throws IOException {
    StringWriter writer = new StringWriter();
    JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
    JobSnapshottingSettings settings = graph.getJobSnapshottingSettings();
    if (settings == null) {
        return "{}";
    }
    gen.writeStartObject();
    {
        gen.writeStringField("mode", settings.isExactlyOnce() ? "exactly_once" : "at_least_once");
        gen.writeNumberField("interval", settings.getCheckpointInterval());
        gen.writeNumberField("timeout", settings.getCheckpointTimeout());
        gen.writeNumberField("min_pause", settings.getMinPauseBetweenCheckpoints());
        gen.writeNumberField("max_concurrent", settings.getMaxConcurrentCheckpoints());
        ExternalizedCheckpointSettings externalization = settings.getExternalizedCheckpointSettings();
        gen.writeObjectFieldStart("externalization");
        {
            if (externalization.externalizeCheckpoints()) {
                gen.writeBooleanField("enabled", true);
                gen.writeBooleanField("delete_on_cancellation", externalization.deleteOnCancellation());
            } else {
                gen.writeBooleanField("enabled", false);
            }
        }
        gen.writeEndObject();
    }
    gen.writeEndObject();
    gen.close();
    return writer.toString();
}

Example 7 with JobSnapshottingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings in project flink by apache.

the class CheckpointConfigHandlerTest method testSimpleConfig.

/**
	 * Tests a simple config.
	 */
@Test
public void testSimpleConfig() throws Exception {
    GraphAndSettings graphAndSettings = createGraphAndSettings(false, true);
    AccessExecutionGraph graph = graphAndSettings.graph;
    JobSnapshottingSettings settings = graphAndSettings.snapshottingSettings;
    CheckpointConfigHandler handler = new CheckpointConfigHandler(mock(ExecutionGraphHolder.class));
    String json = handler.handleRequest(graph, Collections.<String, String>emptyMap());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(json);
    assertEquals("exactly_once", rootNode.get("mode").asText());
    assertEquals(settings.getCheckpointInterval(), rootNode.get("interval").asLong());
    assertEquals(settings.getCheckpointTimeout(), rootNode.get("timeout").asLong());
    assertEquals(settings.getMinPauseBetweenCheckpoints(), rootNode.get("min_pause").asLong());
    assertEquals(settings.getMaxConcurrentCheckpoints(), rootNode.get("max_concurrent").asInt());
    JsonNode externalizedNode = rootNode.get("externalization");
    assertNotNull(externalizedNode);
    assertEquals(false, externalizedNode.get("enabled").asBoolean());
}

Example 8 with JobSnapshottingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings in project flink by apache.

the class CheckpointConfigHandlerTest method testArchiver.

@Test
public void testArchiver() throws IOException {
    JsonArchivist archivist = new CheckpointConfigHandler.CheckpointConfigJsonArchivist();
    GraphAndSettings graphAndSettings = createGraphAndSettings(true, true);
    AccessExecutionGraph graph = graphAndSettings.graph;
    when(graph.getJobID()).thenReturn(new JobID());
    JobSnapshottingSettings settings = graphAndSettings.snapshottingSettings;
    ExternalizedCheckpointSettings externalizedSettings = graphAndSettings.externalizedSettings;
    Collection<ArchivedJson> archives = archivist.archiveJsonWithPath(graph);
    Assert.assertEquals(1, archives.size());
    ArchivedJson archive = archives.iterator().next();
    Assert.assertEquals("/jobs/" + graph.getJobID() + "/checkpoints/config", archive.getPath());
    ObjectMapper mapper = new ObjectMapper();
    JsonNode rootNode = mapper.readTree(archive.getJson());
    Assert.assertEquals("exactly_once", rootNode.get("mode").asText());
    Assert.assertEquals(settings.getCheckpointInterval(), rootNode.get("interval").asLong());
    Assert.assertEquals(settings.getCheckpointTimeout(), rootNode.get("timeout").asLong());
    Assert.assertEquals(settings.getMinPauseBetweenCheckpoints(), rootNode.get("min_pause").asLong());
    Assert.assertEquals(settings.getMaxConcurrentCheckpoints(), rootNode.get("max_concurrent").asInt());
    JsonNode externalizedNode = rootNode.get("externalization");
    Assert.assertNotNull(externalizedNode);
    Assert.assertEquals(externalizedSettings.externalizeCheckpoints(), externalizedNode.get("enabled").asBoolean());
    Assert.assertEquals(externalizedSettings.deleteOnCancellation(), externalizedNode.get("delete_on_cancellation").asBoolean());
}

Example 9 with JobSnapshottingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings in project flink by apache.

the class CheckpointStatsTrackerTest method testGetSnapshottingSettings.

/**
	 * Tests access to the snapshotting settings.
	 */
@Test
public void testGetSnapshottingSettings() throws Exception {
    ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class);
    when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID());
    when(jobVertex.getParallelism()).thenReturn(1);
    JobSnapshottingSettings snapshottingSettings = new JobSnapshottingSettings(Collections.singletonList(new JobVertexID()), Collections.singletonList(new JobVertexID()), Collections.singletonList(new JobVertexID()), 181238123L, 19191992L, 191929L, 123, ExternalizedCheckpointSettings.none(), null, false);
    CheckpointStatsTracker tracker = new CheckpointStatsTracker(0, Collections.singletonList(jobVertex), snapshottingSettings, new UnregisteredMetricsGroup());
    assertEquals(snapshottingSettings, tracker.getSnapshottingSettings());
}

Example 10 with JobSnapshottingSettings

use of org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings in project flink by apache.

the class CoordinatorShutdownTest method testCoordinatorShutsDownOnFailure.

@Test
public void testCoordinatorShutsDownOnFailure() {
    LocalFlinkMiniCluster cluster = null;
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
        cluster = new LocalFlinkMiniCluster(config, true);
        cluster.start();
        // build a test graph with snapshotting enabled
        JobVertex vertex = new JobVertex("Test Vertex");
        vertex.setInvokableClass(FailingBlockingInvokable.class);
        List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
        JobGraph testGraph = new JobGraph("test job", vertex);
        testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000, 60000, 0L, Integer.MAX_VALUE, ExternalizedCheckpointSettings.none(), null, true));
        ActorGateway jmGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
        FiniteDuration timeout = new FiniteDuration(60, TimeUnit.SECONDS);
        JobManagerMessages.SubmitJob submitMessage = new JobManagerMessages.SubmitJob(testGraph, ListeningBehaviour.EXECUTION_RESULT);
        // submit is successful, but then the job blocks due to the invokable
        Future<Object> submitFuture = jmGateway.ask(submitMessage, timeout);
        Await.result(submitFuture, timeout);
        // get the execution graph and store the ExecutionGraph reference
        Future<Object> jobRequestFuture = jmGateway.ask(new JobManagerMessages.RequestJob(testGraph.getJobID()), timeout);
        ExecutionGraph graph = (ExecutionGraph) ((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
        assertNotNull(graph);
        FailingBlockingInvokable.unblock();
        graph.waitUntilFinished();
        // verify that the coordinator was shut down
        CheckpointCoordinator coord = graph.getCheckpointCoordinator();
        assertTrue(coord == null || coord.isShutdown());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
            cluster.awaitTermination();
        }
    }
}