Examples with ClusterClient org.apache.flink.client.program.ClusterClient used on opensource projects

Example 11 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class WritableSavepointITCase method validateModification.

private void validateModification(StateBackend backend, String savepointPath) throws Exception {
    StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    sEnv.setStateBackend(backend);
    DataStream<Account> stream = sEnv.fromCollection(accounts).keyBy(acc -> acc.id).flatMap(new UpdateAndGetAccount()).uid(ACCOUNT_UID);
    CompletableFuture<Collection<Account>> results = collector.collect(stream);
    stream.map(acc -> acc.id).map(new StatefulOperator()).uid(MODIFY_UID).addSink(new DiscardingSink<>());
    JobGraph jobGraph = sEnv.getStreamGraph().getJobGraph();
    jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
    ClusterClient<?> client = MINI_CLUSTER_RESOURCE.getClusterClient();
    Optional<SerializedThrowable> serializedThrowable = client.submitJob(jobGraph).thenCompose(client::requestJobResult).get().getSerializedThrowable();
    Assert.assertFalse(serializedThrowable.isPresent());
    Assert.assertEquals("Unexpected output", 3, results.get().size());
}

Example 12 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class SavepointWriterITCase method validateModification.

private void validateModification(StateBackend backend, String savepointPath) throws Exception {
    StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    if (backend != null) {
        sEnv.setStateBackend(backend);
    }
    DataStream<Account> stream = sEnv.fromCollection(accounts).keyBy(acc -> acc.id).flatMap(new UpdateAndGetAccount()).uid(ACCOUNT_UID);
    CompletableFuture<Collection<Account>> results = collector.collect(stream);
    stream.map(acc -> acc.id).map(new StatefulOperator()).uid(MODIFY_UID).addSink(new DiscardingSink<>());
    JobGraph jobGraph = sEnv.getStreamGraph().getJobGraph();
    jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
    ClusterClient<?> client = MINI_CLUSTER_RESOURCE.getClusterClient();
    Optional<SerializedThrowable> serializedThrowable = client.submitJob(jobGraph).thenCompose(client::requestJobResult).get().getSerializedThrowable();
    Assert.assertFalse(serializedThrowable.isPresent());
    Assert.assertEquals("Unexpected output", 3, results.get().size());
}

Example 13 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class SavepointITCase method testCanRestoreWithModifiedStatelessOperators.

/**
 * FLINK-5985
 *
 * <p>This test ensures we can restore from a savepoint under modifications to the job graph
 * that only concern stateless operators.
 */
@Test
public void testCanRestoreWithModifiedStatelessOperators() throws Exception {
    // Config
    int numTaskManagers = 2;
    int numSlotsPerTaskManager = 2;
    int parallelism = 2;
    // Test deadline
    final Deadline deadline = Deadline.now().plus(Duration.ofMinutes(5));
    // Flink configuration
    final Configuration config = new Configuration();
    config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
    String savepointPath;
    LOG.info("Flink configuration: " + config + ".");
    // Start Flink
    MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(config).setNumberTaskManagers(numTaskManagers).setNumberSlotsPerTaskManager(numSlotsPerTaskManager).build());
    LOG.info("Shutting down Flink cluster.");
    cluster.before();
    ClusterClient<?> client = cluster.getClusterClient();
    try {
        final StatefulCounter statefulCounter = new StatefulCounter();
        StatefulCounter.resetForTest(parallelism);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(parallelism);
        env.addSource(new InfiniteTestSource()).shuffle().map(value -> 4 * value).shuffle().map(statefulCounter).uid("statefulCounter").shuffle().map(value -> 2 * value).addSink(new DiscardingSink<>());
        JobGraph originalJobGraph = env.getStreamGraph().getJobGraph();
        JobID jobID = client.submitJob(originalJobGraph).get();
        // wait for the Tasks to be ready
        waitForAllTaskRunning(cluster.getMiniCluster(), jobID, false);
        assertTrue(StatefulCounter.getProgressLatch().await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
        savepointPath = client.triggerSavepoint(jobID, null, SavepointFormatType.CANONICAL).get();
        LOG.info("Retrieved savepoint: " + savepointPath + ".");
    } finally {
        // Shut down the Flink cluster (thereby canceling the job)
        LOG.info("Shutting down Flink cluster.");
        cluster.after();
    }
    // create a new MiniCluster to make sure we start with completely
    // new resources
    cluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder().setConfiguration(config).setNumberTaskManagers(numTaskManagers).setNumberSlotsPerTaskManager(numSlotsPerTaskManager).build());
    LOG.info("Restarting Flink cluster.");
    cluster.before();
    client = cluster.getClusterClient();
    try {
        // Reset static test helpers
        StatefulCounter.resetForTest(parallelism);
        // Gather all task deployment descriptors
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(parallelism);
        // generate a modified job graph that adds a stateless op
        env.addSource(new InfiniteTestSource()).shuffle().map(new StatefulCounter()).uid("statefulCounter").shuffle().map(value -> value).addSink(new DiscardingSink<>());
        JobGraph modifiedJobGraph = env.getStreamGraph().getJobGraph();
        // Set the savepoint path
        modifiedJobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
        LOG.info("Resubmitting job " + modifiedJobGraph.getJobID() + " with " + "savepoint path " + savepointPath + " in detached mode.");
        // Submit the job
        client.submitJob(modifiedJobGraph).get();
        // Await state is restored
        assertTrue(StatefulCounter.getRestoreLatch().await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
        // Await some progress after restore
        assertTrue(StatefulCounter.getProgressLatch().await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
    } finally {
        cluster.after();
    }
}

Example 14 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class NotifyCheckpointAbortedITCase method testNotifyCheckpointAborted.

/**
 * Verify operators would be notified as checkpoint aborted.
 *
 * <p>The job would run with at least two checkpoints. The 1st checkpoint would fail due to add
 * checkpoint to store, and the 2nd checkpoint would decline by async checkpoint phase of
 * 'DeclineSink'.
 *
 * <p>The job graph looks like: NormalSource --> keyBy --> NormalMap --> DeclineSink
 */
@Test(timeout = TEST_TIMEOUT)
public void testNotifyCheckpointAborted() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(200, CheckpointingMode.EXACTLY_ONCE);
    env.getCheckpointConfig().enableUnalignedCheckpoints(unalignedCheckpointEnabled);
    env.getCheckpointConfig().setTolerableCheckpointFailureNumber(1);
    env.disableOperatorChaining();
    env.setParallelism(1);
    final StateBackend failingStateBackend = new DeclineSinkFailingStateBackend(checkpointPath);
    env.setStateBackend(failingStateBackend);
    env.addSource(new NormalSource()).name("NormalSource").keyBy((KeySelector<Tuple2<Integer, Integer>, Integer>) value -> value.f0).transform("NormalMap", TypeInformation.of(Integer.class), new NormalMap()).transform(DECLINE_SINK_NAME, TypeInformation.of(Object.class), new DeclineSink());
    final ClusterClient<?> clusterClient = cluster.getClusterClient();
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    JobID jobID = jobGraph.getJobID();
    clusterClient.submitJob(jobGraph).get();
    TestingCompletedCheckpointStore.addCheckpointLatch.await();
    log.info("The checkpoint to abort is ready to add to checkpoint store.");
    TestingCompletedCheckpointStore.abortCheckpointLatch.trigger();
    log.info("Verifying whether all operators have been notified of checkpoint-1 aborted.");
    verifyAllOperatorsNotifyAborted();
    log.info("Verified that all operators have been notified of checkpoint-1 aborted.");
    resetAllOperatorsNotifyAbortedLatches();
    verifyAllOperatorsNotifyAbortedTimes(1);
    NormalSource.waitLatch.trigger();
    log.info("Verifying whether all operators have been notified of checkpoint-2 aborted.");
    verifyAllOperatorsNotifyAborted();
    log.info("Verified that all operators have been notified of checkpoint-2 aborted.");
    verifyAllOperatorsNotifyAbortedTimes(2);
    clusterClient.cancel(jobID).get();
    log.info("Test is verified successfully as expected.");
}

Example 15 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class CliFrontend method createClient.

/**
	 * Creates a {@link ClusterClient} object from the given command line options and other parameters.
	 * @param options Command line options
	 * @param program The program for which to create the client.
	 * @throws Exception
	 */
protected ClusterClient createClient(CommandLineOptions options, PackagedProgram program) throws Exception {
    // Get the custom command-line (e.g. Standalone/Yarn/Mesos)
    CustomCommandLine<?> activeCommandLine = getActiveCustomCommandLine(options.getCommandLine());
    ClusterClient client;
    try {
        client = activeCommandLine.retrieveCluster(options.getCommandLine(), config);
        logAndSysout("Cluster configuration: " + client.getClusterIdentifier());
    } catch (UnsupportedOperationException e) {
        try {
            String applicationName = "Flink Application: " + program.getMainClassName();
            client = activeCommandLine.createCluster(applicationName, options.getCommandLine(), config, program.getAllLibraries());
            logAndSysout("Cluster started: " + client.getClusterIdentifier());
        } catch (UnsupportedOperationException e2) {
            throw new IllegalConfigurationException("The JobManager address is neither provided at the command-line, " + "nor configured in flink-conf.yaml.");
        }
    }
    // Avoid resolving the JobManager Gateway here to prevent blocking until we invoke the user's program.
    final InetSocketAddress jobManagerAddress = client.getJobManagerAddress();
    logAndSysout("Using address " + jobManagerAddress.getHostString() + ":" + jobManagerAddress.getPort() + " to connect to JobManager.");
    logAndSysout("JobManager web interface address " + client.getWebInterfaceURL());
    return client;
}