Examples with ClusterClient org.apache.flink.client.program.ClusterClient used on opensource projects

Example 6 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class YARNHighAvailabilityITCase method testMultipleAMKill.

/**
	 * Tests that the application master can be killed multiple times and that the surviving
	 * TaskManager successfully reconnects to the newly started JobManager.
	 * @throws Exception
	 */
@Test
public void testMultipleAMKill() throws Exception {
    final int numberKillingAttempts = numberApplicationAttempts - 1;
    TestingYarnClusterDescriptor flinkYarnClient = new TestingYarnClusterDescriptor();
    Assert.assertNotNull("unable to get yarn client", flinkYarnClient);
    flinkYarnClient.setTaskManagerCount(1);
    flinkYarnClient.setJobManagerMemory(768);
    flinkYarnClient.setTaskManagerMemory(1024);
    flinkYarnClient.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
    flinkYarnClient.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
    String confDirPath = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
    flinkYarnClient.setConfigurationDirectory(confDirPath);
    String fsStateHandlePath = temp.getRoot().getPath();
    // load the configuration
    File configDirectory = new File(confDirPath);
    GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
    flinkYarnClient.setFlinkConfiguration(GlobalConfiguration.loadConfiguration());
    flinkYarnClient.setDynamicPropertiesEncoded("recovery.mode=zookeeper@@recovery.zookeeper.quorum=" + zkServer.getConnectString() + "@@yarn.application-attempts=" + numberApplicationAttempts + "@@" + CoreOptions.STATE_BACKEND + "=FILESYSTEM" + "@@" + FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY + "=" + fsStateHandlePath + "/checkpoints" + "@@" + HighAvailabilityOptions.HA_STORAGE_PATH.key() + "=" + fsStateHandlePath + "/recovery");
    flinkYarnClient.setConfigurationFilePath(new Path(confDirPath + File.separator + "flink-conf.yaml"));
    ClusterClient yarnCluster = null;
    final FiniteDuration timeout = new FiniteDuration(2, TimeUnit.MINUTES);
    try {
        yarnCluster = flinkYarnClient.deploy();
        final Configuration config = yarnCluster.getFlinkConfiguration();
        new JavaTestKit(actorSystem) {

            {
                for (int attempt = 0; attempt < numberKillingAttempts; attempt++) {
                    new Within(timeout) {

                        @Override
                        protected void run() {
                            try {
                                LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);
                                ActorGateway gateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
                                ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway.leaderSessionID());
                                gateway.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
                                expectMsgEquals(Acknowledge.get());
                                gateway.tell(PoisonPill.getInstance());
                            } catch (Exception e) {
                                throw new AssertionError("Could not complete test.", e);
                            }
                        }
                    };
                }
                new Within(timeout) {

                    @Override
                    protected void run() {
                        try {
                            LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);
                            ActorGateway gateway2 = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, actorSystem, timeout);
                            ActorGateway selfGateway = new AkkaActorGateway(getRef(), gateway2.leaderSessionID());
                            gateway2.tell(new TestingJobManagerMessages.NotifyWhenAtLeastNumTaskManagerAreRegistered(1), selfGateway);
                            expectMsgEquals(Acknowledge.get());
                        } catch (Exception e) {
                            throw new AssertionError("Could not complete test.", e);
                        }
                    }
                };
            }
        };
    } finally {
        if (yarnCluster != null) {
            yarnCluster.shutdown();
        }
    }
}

Example 7 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class YARNSessionFIFOITCase method testJavaAPI.

/**
	 * Test the YARN Java API
	 */
@Test
public void testJavaAPI() {
    final int WAIT_TIME = 15;
    LOG.info("Starting testJavaAPI()");
    AbstractYarnClusterDescriptor flinkYarnClient = new YarnClusterDescriptor();
    Assert.assertNotNull("unable to get yarn client", flinkYarnClient);
    flinkYarnClient.setTaskManagerCount(1);
    flinkYarnClient.setJobManagerMemory(768);
    flinkYarnClient.setTaskManagerMemory(1024);
    flinkYarnClient.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
    flinkYarnClient.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
    String confDirPath = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
    flinkYarnClient.setConfigurationDirectory(confDirPath);
    flinkYarnClient.setFlinkConfiguration(GlobalConfiguration.loadConfiguration());
    flinkYarnClient.setConfigurationFilePath(new Path(confDirPath + File.separator + "flink-conf.yaml"));
    // deploy
    ClusterClient yarnCluster = null;
    try {
        yarnCluster = flinkYarnClient.deploy();
    } catch (Exception e) {
        LOG.warn("Failing test", e);
        Assert.fail("Error while deploying YARN cluster: " + e.getMessage());
    }
    GetClusterStatusResponse expectedStatus = new GetClusterStatusResponse(1, 1);
    for (int second = 0; second < WAIT_TIME * 2; second++) {
        // run "forever"
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Interrupted", e);
        }
        GetClusterStatusResponse status = yarnCluster.getClusterStatus();
        if (status != null && status.equals(expectedStatus)) {
            LOG.info("ClusterClient reached status " + status);
            // all good, cluster started
            break;
        }
        if (second > WAIT_TIME) {
            // we waited for 15 seconds. cluster didn't come up correctly
            Assert.fail("The custer didn't start after " + WAIT_TIME + " seconds");
        }
    }
    // use the cluster
    Assert.assertNotNull(yarnCluster.getJobManagerAddress());
    Assert.assertNotNull(yarnCluster.getWebInterfaceURL());
    LOG.info("Shutting down cluster. All tests passed");
    // shutdown cluster
    yarnCluster.shutdown();
    LOG.info("Finished testJavaAPI()");
}

Example 8 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class RemoteStreamEnvironment method executeRemotely.

/**
	 * Executes the remote job.
	 * 
	 * @param streamGraph
	 *            Stream Graph to execute
	 * @param jarFiles
	 * 			  List of jar file URLs to ship to the cluster
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException {
    if (LOG.isInfoEnabled()) {
        LOG.info("Running remotely at {}:{}", host, port);
    }
    ClassLoader usercodeClassLoader = JobWithJars.buildUserCodeClassLoader(jarFiles, globalClasspaths, getClass().getClassLoader());
    Configuration configuration = new Configuration();
    configuration.addAll(this.clientConfiguration);
    configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, host);
    configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);
    ClusterClient client;
    try {
        client = new StandaloneClusterClient(configuration);
        client.setPrintStatusDuringExecution(getConfig().isSysoutLoggingEnabled());
    } catch (Exception e) {
        throw new ProgramInvocationException("Cannot establish connection to JobManager: " + e.getMessage(), e);
    }
    try {
        return client.run(streamGraph, jarFiles, globalClasspaths, usercodeClassLoader).getJobExecutionResult();
    } catch (ProgramInvocationException e) {
        throw e;
    } catch (Exception e) {
        String term = e.getMessage() == null ? "." : (": " + e.getMessage());
        throw new ProgramInvocationException("The program execution failed" + term, e);
    } finally {
        client.shutdown();
    }
}

Example 9 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class CliFrontendSavepointTest method testDisposeSavepointSuccess.

// ------------------------------------------------------------------------
// Dispose savepoint
// ------------------------------------------------------------------------
@Test
public void testDisposeSavepointSuccess() throws Exception {
    replaceStdOutAndStdErr();
    String savepointPath = "expectedSavepointPath";
    ClusterClient clusterClient = new DisposeSavepointClusterClient((String path) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration());
    try {
        CliFrontend frontend = new MockedCliFrontend(clusterClient);
        String[] parameters = { "-d", savepointPath };
        frontend.savepoint(parameters);
        String outMsg = buffer.toString();
        assertTrue(outMsg.contains(savepointPath));
        assertTrue(outMsg.contains("disposed"));
    } finally {
        clusterClient.close();
        restoreStdOutAndStdErr();
    }
}

Example 10 with ClusterClient

use of org.apache.flink.client.program.ClusterClient in project flink by apache.

the class AbstractSessionClusterExecutor method execute.

@Override
public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration, @Nonnull final ClassLoader userCodeClassloader) throws Exception {
    final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);
    try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
        final ClusterID clusterID = clusterClientFactory.getClusterId(configuration);
        checkState(clusterID != null);
        final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor.retrieve(clusterID);
        ClusterClient<ClusterID> clusterClient = clusterClientProvider.getClusterClient();
        return clusterClient.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction(jobId -> {
            ClientUtils.waitUntilJobInitializationFinished(() -> clusterClient.getJobStatus(jobId).get(), () -> clusterClient.requestJobResult(jobId).get(), userCodeClassloader);
            return jobId;
        })).thenApplyAsync(jobID -> (JobClient) new ClusterClientJobClientAdapter<>(clusterClientProvider, jobID, userCodeClassloader)).whenCompleteAsync((ignored1, ignored2) -> clusterClient.close());
    }
}