Examples with JobClient org.apache.flink.core.execution.JobClient used on opensource projects

Example 56 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class LocalExecutorITCase method testLocalExecutorWithWordCount.

@Test(timeout = 60_000)
public void testLocalExecutorWithWordCount() throws InterruptedException {
    try {
        // set up the files
        File inFile = File.createTempFile("wctext", ".in");
        File outFile = File.createTempFile("wctext", ".out");
        inFile.deleteOnExit();
        outFile.deleteOnExit();
        try (FileWriter fw = new FileWriter(inFile)) {
            fw.write(WordCountData.TEXT);
        }
        final Configuration config = new Configuration();
        config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
        config.setBoolean(DeploymentOptions.ATTACHED, true);
        Plan wcPlan = getWordCountPlan(inFile, outFile, parallelism);
        wcPlan.setExecutionConfig(new ExecutionConfig());
        JobClient jobClient = executor.execute(wcPlan, config, ClassLoader.getSystemClassLoader()).get();
        jobClient.getJobExecutionResult().get();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
    assertThat(miniCluster.isRunning(), is(false));
}

Example 57 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class RestoreUpgradedJobITCase method runOriginalJob.

@NotNull
private String runOriginalJob() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getCheckpointConfig().setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
    env.getCheckpointConfig().enableUnalignedCheckpoints(false);
    env.getCheckpointConfig().setCheckpointStorage("file://" + temporaryFolder.getRoot().getAbsolutePath());
    env.setParallelism(PARALLELISM);
    env.enableCheckpointing(Integer.MAX_VALUE);
    // Different order of maps before and after savepoint.
    env.addSource(new IntSource(allDataEmittedLatch)).map(new IntMap(MAP_5.id())).uid(MAP_5.name()).forward().map(new IntMap(MAP_1.id())).uid(MAP_1.name()).slotSharingGroup("anotherSharingGroup").keyBy((key) -> key).map(new IntMap(MAP_6.id())).uid(MAP_6.name()).rebalance().map(new IntMap(MAP_4.id())).uid(MAP_4.name()).broadcast().map(new IntMap(MAP_2.id())).uid(MAP_2.name()).rescale().map(new IntMap(MAP_3.id())).uid(MAP_3.name()).addSink(new IntSink(result)).setParallelism(1);
    // when: Job is executed.
    JobClient jobClient = env.executeAsync("Total sum");
    waitForAllTaskRunning(CLUSTER.getMiniCluster(), jobClient.getJobID(), false);
    allDataEmittedLatch.get().await();
    allDataEmittedLatch.get().reset();
    return stopWithSnapshot(jobClient);
}

Example 58 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class UnalignedCheckpointCompatibilityITCase method runAndTakeSavepoint.

private Tuple2<String, Map<String, Object>> runAndTakeSavepoint() throws Exception {
    JobClient jobClient = submitJobInitially(env(startAligned, 0));
    waitForAllTaskRunning(() -> miniCluster.getMiniCluster().getExecutionGraph(jobClient.getJobID()).get(), false);
    // wait for some backpressure from sink
    Thread.sleep(FIRST_RUN_BACKPRESSURE_MS);
    Future<Map<String, Object>> accFuture = jobClient.getJobExecutionResult().thenApply(JobExecutionResult::getAllAccumulatorResults);
    Future<String> savepointFuture = jobClient.stopWithSavepoint(false, tempFolder().toURI().toString(), SavepointFormatType.CANONICAL);
    return new Tuple2<>(savepointFuture.get(), accFuture.get());
}

Example 59 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class CheckpointStoreITCase method testJobClientRemainsResponsiveDuringCompletedCheckpointStoreRecovery.

@Test
public void testJobClientRemainsResponsiveDuringCompletedCheckpointStoreRecovery() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(10);
    env.setRestartStrategy(fixedDelayRestart(2, /* failure on processing + on recovery */
    0));
    env.addSource(emitUntil(() -> FailingMapper.failedAndProcessed)).map(new FailingMapper()).addSink(new DiscardingSink<>());
    final JobClient jobClient = env.executeAsync();
    BlockingHighAvailabilityServiceFactory.fetchRemoteCheckpointsStart.await();
    for (int i = 0; i < 10; i++) {
        final JobStatus jobStatus = jobClient.getJobStatus().get();
        assertEquals(JobStatus.INITIALIZING, jobStatus);
    }
    BlockingHighAvailabilityServiceFactory.fetchRemoteCheckpointsFinished.countDown();
    // Await for job to finish.
    jobClient.getJobExecutionResult().get();
    checkState(FailingMapper.failedAndProcessed);
}

Example 60 with JobClient

use of org.apache.flink.core.execution.JobClient in project flink by apache.

the class TestUtils method tryExecute.

/**
 * Execute the job and wait for the job result synchronously.
 *
 * @throws Exception If executing the environment throws an exception which does not have {@link
 *     SuccessException} as a cause.
 */
public static void tryExecute(StreamExecutionEnvironment see, String name) throws Exception {
    JobClient jobClient = null;
    try {
        StreamGraph graph = see.getStreamGraph();
        graph.setJobName(name);
        jobClient = see.executeAsync(graph);
        jobClient.getJobExecutionResult().get();
    } catch (Throwable root) {
        if (jobClient != null) {
            try {
                jobClient.cancel().get();
            } catch (Exception e) {
            // Exception could be thrown if the job has already finished.
            // Ignore the exception.
            }
        }
        Optional<SuccessException> successAsCause = ExceptionUtils.findThrowable(root, SuccessException.class);
        if (!successAsCause.isPresent()) {
            root.printStackTrace();
            fail("Test failed: " + root.getMessage());
        }
    }
}