Examples with TestingCluster org.apache.flink.runtime.testingUtils.TestingCluster used on opensource projects

Example 1 with TestingCluster

use of org.apache.flink.runtime.testingUtils.TestingCluster in project flink by apache.

the class JobRetrievalITCase method before.

@BeforeClass
public static void before() {
    cluster = new TestingCluster(new Configuration(), false);
    cluster.start();
}

Example 2 with TestingCluster

use of org.apache.flink.runtime.testingUtils.TestingCluster in project flink by apache.

the class JobClientActorRecoveryITCase method testJobClientRecovery.

/**
	 * Tests wether the JobClientActor can connect to a newly elected leading job manager to obtain
	 * the JobExecutionResult. The submitted job blocks for the first execution attempt. The
	 * leading job manager will be killed so that the second job manager will be elected as the
	 * leader. The newly elected leader has to retrieve the checkpointed job from ZooKeeper
	 * and continue its execution. This time, the job does not block and, thus, can be finished.
	 * The execution result should be sent to the JobClientActor which originally submitted the
	 * job.
	 *
	 * @throws Exception
	 */
@Test
public void testJobClientRecovery() throws Exception {
    File rootFolder = tempFolder.getRoot();
    Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(zkServer.getConnectString(), rootFolder.getPath());
    config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 2);
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
    final TestingCluster cluster = new TestingCluster(config);
    cluster.start();
    JobVertex blockingVertex = new JobVertex("Blocking Vertex");
    blockingVertex.setInvokableClass(BlockingTask.class);
    blockingVertex.setParallelism(1);
    final JobGraph jobGraph = new JobGraph("Blocking Test Job", blockingVertex);
    final Promise<JobExecutionResult> promise = new scala.concurrent.impl.Promise.DefaultPromise<>();
    Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow();
    try {
        Thread submitter = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    JobExecutionResult result = cluster.submitJobAndWait(jobGraph, false);
                    promise.success(result);
                } catch (Exception e) {
                    promise.failure(e);
                }
            }
        });
        submitter.start();
        synchronized (BlockingTask.waitLock) {
            while (BlockingTask.HasBlockedExecution < 1 && deadline.hasTimeLeft()) {
                BlockingTask.waitLock.wait(deadline.timeLeft().toMillis());
            }
        }
        if (deadline.isOverdue()) {
            Assert.fail("The job has not blocked within the given deadline.");
        }
        ActorGateway gateway = cluster.getLeaderGateway(deadline.timeLeft());
        gateway.tell(TestingJobManagerMessages.getDisablePostStop());
        gateway.tell(PoisonPill.getInstance());
        // if the job fails then an exception is thrown here
        Await.result(promise.future(), deadline.timeLeft());
    } finally {
        cluster.shutdown();
    }
}

Example 3 with TestingCluster

use of org.apache.flink.runtime.testingUtils.TestingCluster in project flink by apache.

the class WebRuntimeMonitorITCase method testNoCopyFromJar.

/**
	 * Files are copied from the flink-dist jar to a temporary directory and
	 * then served from there. Only allow to copy files from <code>flink-dist.jar:/web</code>
	 */
@Test
public void testNoCopyFromJar() throws Exception {
    final Deadline deadline = TestTimeout.fromNow();
    TestingCluster flink = null;
    WebRuntimeMonitor webMonitor = null;
    try {
        flink = new TestingCluster(new Configuration());
        flink.start(true);
        webMonitor = startWebRuntimeMonitor(flink);
        try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
            String expectedIndex = new Scanner(new File(MAIN_RESOURCES_PATH + "/index.html")).useDelimiter("\\A").next();
            // 1) Request index.html from web server
            client.sendGetRequest("index.html", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("html"));
            assertEquals(expectedIndex, response.getContent());
            // 2) Request file from class loader
            client.sendGetRequest("../log4j-test.properties", deadline.timeLeft());
            response = client.getNextResponse(deadline.timeLeft());
            assertEquals("Returned status code " + response.getStatus() + " for file outside of web root.", HttpResponseStatus.NOT_FOUND, response.getStatus());
            assertFalse("Did not respond with the file, but still copied it from the JAR.", new File(webMonitor.getBaseDir(new Configuration()), "log4j-test.properties").exists());
            // 3) Request non-existing file
            client.sendGetRequest("not-existing-resource", deadline.timeLeft());
            response = client.getNextResponse(deadline.timeLeft());
            assertEquals("Unexpected status code " + response.getStatus() + " for file outside of web root.", HttpResponseStatus.NOT_FOUND, response.getStatus());
        }
    } finally {
        if (flink != null) {
            flink.shutdown();
        }
        if (webMonitor != null) {
            webMonitor.stop();
        }
    }
}

Example 4 with TestingCluster

use of org.apache.flink.runtime.testingUtils.TestingCluster in project flink by apache.

the class WebRuntimeMonitorITCase method testStandaloneWebRuntimeMonitor.

/**
	 * Tests operation of the monitor in standalone operation.
	 */
@Test
public void testStandaloneWebRuntimeMonitor() throws Exception {
    final Deadline deadline = TestTimeout.fromNow();
    TestingCluster flink = null;
    WebRuntimeMonitor webMonitor = null;
    try {
        // Flink w/o a web monitor
        flink = new TestingCluster(new Configuration());
        flink.start(true);
        webMonitor = startWebRuntimeMonitor(flink);
        try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
            String expected = new Scanner(new File(MAIN_RESOURCES_PATH + "/index.html")).useDelimiter("\\A").next();
            // Request the file from the web server
            client.sendGetRequest("index.html", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("html"));
            assertEquals(expected, response.getContent());
            // Simple overview request
            client.sendGetRequest("/overview", deadline.timeLeft());
            response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("json"));
            assertTrue(response.getContent().contains("\"taskmanagers\":1"));
        }
    } finally {
        if (flink != null) {
            flink.shutdown();
        }
        if (webMonitor != null) {
            webMonitor.stop();
        }
    }
}

Example 5 with TestingCluster

use of org.apache.flink.runtime.testingUtils.TestingCluster in project flink by apache.

the class JobManagerTest method testRequestPartitionStateUnregisteredExecution.

/**
	 * Tests the JobManager response when the execution is not registered with
	 * the ExecutionGraph.
	 */
@Test
public void testRequestPartitionStateUnregisteredExecution() throws Exception {
    new JavaTestKit(system) {

        {
            new Within(duration("15 seconds")) {

                @Override
                protected void run() {
                    // Setup
                    TestingCluster cluster = null;
                    try {
                        cluster = startTestingCluster(4, 1, DEFAULT_AKKA_ASK_TIMEOUT());
                        final IntermediateDataSetID rid = new IntermediateDataSetID();
                        // Create a task
                        final JobVertex sender = new JobVertex("Sender");
                        sender.setParallelism(1);
                        // just finish
                        sender.setInvokableClass(NoOpInvokable.class);
                        sender.createAndAddResultDataSet(rid, PIPELINED);
                        final JobVertex sender2 = new JobVertex("Blocking Sender");
                        sender2.setParallelism(1);
                        // just block
                        sender2.setInvokableClass(BlockingNoOpInvokable.class);
                        sender2.createAndAddResultDataSet(new IntermediateDataSetID(), PIPELINED);
                        final JobGraph jobGraph = new JobGraph("Fast finishing producer test job", sender, sender2);
                        final JobID jid = jobGraph.getJobID();
                        final ActorGateway jobManagerGateway = cluster.getLeaderGateway(TestingUtils.TESTING_DURATION());
                        // we can set the leader session ID to None because we don't use this gateway to send messages
                        final ActorGateway testActorGateway = new AkkaActorGateway(getTestActor(), null);
                        // Submit the job and wait for all vertices to be running
                        jobManagerGateway.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT), testActorGateway);
                        expectMsgClass(JobSubmitSuccess.class);
                        jobManagerGateway.tell(new WaitForAllVerticesToBeRunningOrFinished(jid), testActorGateway);
                        expectMsgClass(AllVerticesRunning.class);
                        Future<Object> egFuture = jobManagerGateway.ask(new RequestExecutionGraph(jobGraph.getJobID()), remaining());
                        ExecutionGraphFound egFound = (ExecutionGraphFound) Await.result(egFuture, remaining());
                        ExecutionGraph eg = (ExecutionGraph) egFound.executionGraph();
                        ExecutionVertex vertex = eg.getJobVertex(sender.getID()).getTaskVertices()[0];
                        while (vertex.getExecutionState() != ExecutionState.FINISHED) {
                            Thread.sleep(1);
                        }
                        IntermediateResultPartition partition = vertex.getProducedPartitions().values().iterator().next();
                        ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), vertex.getCurrentExecutionAttempt().getAttemptId());
                        // Producer finished, request state
                        Object request = new RequestPartitionProducerState(jid, rid, partitionId);
                        Future<ExecutionState> producerStateFuture = jobManagerGateway.ask(request, getRemainingTime()).mapTo(ClassTag$.MODULE$.<ExecutionState>apply(ExecutionState.class));
                        assertEquals(ExecutionState.FINISHED, Await.result(producerStateFuture, getRemainingTime()));
                    } catch (Exception e) {
                        e.printStackTrace();
                        fail(e.getMessage());
                    } finally {
                        if (cluster != null) {
                            cluster.shutdown();
                        }
                    }
                }
            };
        }
    };
}