Examples with StandaloneClusterClient org.apache.flink.client.program.StandaloneClusterClient used on opensource projects

Example 6 with StandaloneClusterClient

use of org.apache.flink.client.program.StandaloneClusterClient in project flink by apache.

the class FlinkClient method killTopologyWithOpts.

public void killTopologyWithOpts(final String name, final KillOptions options) throws NotAliveException {
    final JobID jobId = this.getTopologyJobId(name);
    if (jobId == null) {
        throw new NotAliveException("Storm topology with name " + name + " not found.");
    }
    if (options != null) {
        try {
            Thread.sleep(1000 * options.get_wait_secs());
        } catch (final InterruptedException e) {
            throw new RuntimeException(e);
        }
    }
    final Configuration configuration = GlobalConfiguration.loadConfiguration();
    configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, this.jobManagerHost);
    configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, this.jobManagerPort);
    final ClusterClient client;
    try {
        client = new StandaloneClusterClient(configuration);
    } catch (final IOException e) {
        throw new RuntimeException("Could not establish a connection to the job manager", e);
    }
    try {
        client.stop(jobId);
    } catch (final Exception e) {
        throw new RuntimeException("Cannot stop job.", e);
    }
}

Example 7 with StandaloneClusterClient

use of org.apache.flink.client.program.StandaloneClusterClient in project flink by apache.

the class SavepointMigrationTestBase method restoreAndExecute.

@SafeVarargs
protected final void restoreAndExecute(StreamExecutionEnvironment env, String savepointPath, Tuple2<String, Integer>... expectedAccumulators) throws Exception {
    // Retrieve the job manager
    Await.result(cluster.leaderGateway().future(), DEADLINE.timeLeft());
    // Submit the job
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
    JobSubmissionResult jobSubmissionResult = cluster.submitJobDetached(jobGraph);
    StandaloneClusterClient clusterClient = new StandaloneClusterClient(cluster.configuration());
    boolean done = false;
    while (DEADLINE.hasTimeLeft()) {
        Thread.sleep(100);
        Map<String, Object> accumulators = clusterClient.getAccumulators(jobSubmissionResult.getJobID());
        boolean allDone = true;
        for (Tuple2<String, Integer> acc : expectedAccumulators) {
            Integer numFinished = (Integer) accumulators.get(acc.f0);
            if (numFinished == null) {
                allDone = false;
                break;
            }
            if (!numFinished.equals(acc.f1)) {
                allDone = false;
                break;
            }
        }
        if (allDone) {
            done = true;
            break;
        }
    }
    if (!done) {
        fail("Did not see the expected accumulator results within time limit.");
    }
}

Example 8 with StandaloneClusterClient

use of org.apache.flink.client.program.StandaloneClusterClient in project flink by apache.

the class SavepointMigrationTestBase method executeAndSavepoint.

@SafeVarargs
protected final void executeAndSavepoint(StreamExecutionEnvironment env, String savepointPath, Tuple2<String, Integer>... expectedAccumulators) throws Exception {
    // Retrieve the job manager
    ActorGateway jobManager = Await.result(cluster.leaderGateway().future(), DEADLINE.timeLeft());
    // Submit the job
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    JobSubmissionResult jobSubmissionResult = cluster.submitJobDetached(jobGraph);
    LOG.info("Submitted job {} and waiting...", jobSubmissionResult.getJobID());
    StandaloneClusterClient clusterClient = new StandaloneClusterClient(cluster.configuration());
    boolean done = false;
    while (DEADLINE.hasTimeLeft()) {
        Thread.sleep(100);
        Map<String, Object> accumulators = clusterClient.getAccumulators(jobSubmissionResult.getJobID());
        boolean allDone = true;
        for (Tuple2<String, Integer> acc : expectedAccumulators) {
            Integer numFinished = (Integer) accumulators.get(acc.f0);
            if (numFinished == null) {
                allDone = false;
                break;
            }
            if (!numFinished.equals(acc.f1)) {
                allDone = false;
                break;
            }
        }
        if (allDone) {
            done = true;
            break;
        }
    }
    if (!done) {
        fail("Did not see the expected accumulator results within time limit.");
    }
    LOG.info("Triggering savepoint.");
    // Flink 1.2
    final Future<Object> savepointResultFuture = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobSubmissionResult.getJobID(), Option.<String>empty()), DEADLINE.timeLeft());
    // Flink 1.1
    //		final Future<Object> savepointResultFuture =
    //				jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobSubmissionResult.getJobID()), DEADLINE.timeLeft());
    Object savepointResult = Await.result(savepointResultFuture, DEADLINE.timeLeft());
    if (savepointResult instanceof JobManagerMessages.TriggerSavepointFailure) {
        fail("Error drawing savepoint: " + ((JobManagerMessages.TriggerSavepointFailure) savepointResult).cause());
    }
    // jobmanager will store savepoint in heap, we have to retrieve it
    final String jobmanagerSavepointPath = ((JobManagerMessages.TriggerSavepointSuccess) savepointResult).savepointPath();
    LOG.info("Saved savepoint: " + jobmanagerSavepointPath);
    // Flink 1.2
    FileUtils.moveFile(new File(new URI(jobmanagerSavepointPath).getPath()), new File(savepointPath));
// Flink 1.1
// Retrieve the savepoint from the testing job manager
//		LOG.info("Requesting the savepoint.");
//		Future<Object> savepointFuture = jobManager.ask(new TestingJobManagerMessages.RequestSavepoint(jobmanagerSavepointPath), DEADLINE.timeLeft());
//
//		Savepoint savepoint = ((TestingJobManagerMessages.ResponseSavepoint) Await.result(savepointFuture, DEADLINE.timeLeft())).savepoint();
//		LOG.info("Retrieved savepoint: " + jobmanagerSavepointPath + ".");
//
//		LOG.info("Storing savepoint to file.");
//		Configuration config = new Configuration();
//		config.setString(org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
//		config.setString(org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, "file:///Users/aljoscha/Downloads");
//		String path = org.apache.flink.runtime.checkpoint.savepoint.SavepointStoreFactory.createFromConfig(config).storeSavepoint(savepoint);
//
//		FileUtils.moveFile(new File(new URI(path).getPath()), new File(savepointPath));
}

Example 9 with StandaloneClusterClient

use of org.apache.flink.client.program.StandaloneClusterClient in project flink by apache.

the class JobRetrievalITCase method testJobRetrieval.

@Test
public void testJobRetrieval() throws Exception {
    final JobID jobID = new JobID();
    final JobVertex imalock = new JobVertex("imalock");
    imalock.setInvokableClass(SemaphoreInvokable.class);
    final JobGraph jobGraph = new JobGraph(jobID, "testjob", imalock);
    final ClusterClient client = new StandaloneClusterClient(cluster.configuration());
    // acquire the lock to make sure that the job cannot complete until the job client
    // has been attached in resumingThread
    lock.acquire();
    client.runDetached(jobGraph, JobRetrievalITCase.class.getClassLoader());
    final Thread resumingThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                assertNotNull(client.retrieveJob(jobID));
            } catch (Throwable e) {
                fail(e.getMessage());
            }
        }
    });
    final Seq<ActorSystem> actorSystemSeq = cluster.jobManagerActorSystems().get();
    final ActorSystem actorSystem = actorSystemSeq.last();
    JavaTestKit testkit = new JavaTestKit(actorSystem);
    final ActorRef jm = cluster.getJobManagersAsJava().get(0);
    // wait until client connects
    jm.tell(TestingJobManagerMessages.getNotifyWhenClientConnects(), testkit.getRef());
    // confirm registration
    testkit.expectMsgEquals(true);
    // kick off resuming
    resumingThread.start();
    // wait for client to connect
    testkit.expectMsgAllOf(TestingJobManagerMessages.getClientConnected(), TestingJobManagerMessages.getClassLoadingPropsDelivered());
    // client has connected, we can release the lock
    lock.release();
    resumingThread.join();
}