Examples with DispatcherGateway org.apache.flink.runtime.dispatcher.DispatcherGateway used on opensource projects

Example 11 with DispatcherGateway

use of org.apache.flink.runtime.dispatcher.DispatcherGateway in project flink by apache.

the class JobSubmitHandlerTest method testFileHandling.

@Test
public void testFileHandling() throws Exception {
    final String dcEntryName = "entry";
    CompletableFuture<JobGraph> submittedJobGraphFuture = new CompletableFuture<>();
    DispatcherGateway dispatcherGateway = TestingDispatcherGateway.newBuilder().setBlobServerPort(blobServer.getPort()).setSubmitFunction(submittedJobGraph -> {
        submittedJobGraphFuture.complete(submittedJobGraph);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(dispatcherGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    final Path jarFile = TEMPORARY_FOLDER.newFile().toPath();
    final Path artifactFile = TEMPORARY_FOLDER.newFile().toPath();
    final JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    // the entry that should be updated
    jobGraph.addUserArtifact(dcEntryName, new DistributedCache.DistributedCacheEntry("random", false));
    try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
        objectOut.writeObject(jobGraph);
    }
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.singletonList(jarFile.getFileName().toString()), Collections.singleton(new JobSubmitRequestBody.DistributedCacheFile(dcEntryName, artifactFile.getFileName().toString())));
    handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance(), Arrays.asList(jobGraphFile.toFile(), jarFile.toFile(), artifactFile.toFile())), dispatcherGateway).get();
    Assert.assertTrue("No JobGraph was submitted.", submittedJobGraphFuture.isDone());
    final JobGraph submittedJobGraph = submittedJobGraphFuture.get();
    Assert.assertEquals(1, submittedJobGraph.getUserJarBlobKeys().size());
    Assert.assertEquals(1, submittedJobGraph.getUserArtifacts().size());
    Assert.assertNotNull(submittedJobGraph.getUserArtifacts().get(dcEntryName).blobKey);
}

Example 12 with DispatcherGateway

use of org.apache.flink.runtime.dispatcher.DispatcherGateway in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testErrorHandlerIsCalledWhenApplicationStatusIsUnknown.

@Test
public void testErrorHandlerIsCalledWhenApplicationStatusIsUnknown() throws Exception {
    // we're "listening" on this to be completed to verify that the cluster
    // is being shut down from the ApplicationDispatcherBootstrap
    final AtomicBoolean shutdownCalled = new AtomicBoolean(false);
    final TestingDispatcherGateway.Builder dispatcherBuilder = canceledJobGatewayBuilder().setRequestJobResultFunction(jobID -> CompletableFuture.completedFuture(createUnknownJobResult(jobID))).setClusterShutdownFunction(status -> {
        shutdownCalled.set(true);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final TestingDispatcherGateway dispatcherGateway = dispatcherBuilder.build();
    final CompletableFuture<Void> errorHandlerFuture = new CompletableFuture<>();
    final ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3, dispatcherGateway, scheduledExecutor, errorHandlerFuture::completeExceptionally);
    // check that bootstrap shutdown completes exceptionally
    assertException(bootstrap.getApplicationCompletionFuture(), UnsuccessfulExecutionException.class);
    // and exception gets propagated to error handler
    assertException(bootstrap.getApplicationCompletionFuture(), UnsuccessfulExecutionException.class);
    // and cluster didn't shut down
    assertFalse(shutdownCalled.get());
}

Example 13 with DispatcherGateway

use of org.apache.flink.runtime.dispatcher.DispatcherGateway in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testShutdownDisabled.

@ParameterizedTest
@EnumSource(value = JobStatus.class, names = { "FINISHED", "CANCELED", "FAILED" })
public void testShutdownDisabled(JobStatus jobStatus) throws Exception {
    final Configuration configurationUnderTest = getConfiguration();
    configurationUnderTest.set(DeploymentOptions.SHUTDOWN_ON_APPLICATION_FINISH, false);
    final TestingDispatcherGateway dispatcherGateway = dispatcherGatewayBuilder(jobStatus).setClusterShutdownFunction(status -> {
        fail("Cluster shutdown should not be called");
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(configurationUnderTest, dispatcherGateway, scheduledExecutor);
    // Wait until bootstrap is finished to make sure cluster shutdown isn't called
    bootstrap.getBootstrapCompletionFuture().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}

Example 14 with DispatcherGateway

use of org.apache.flink.runtime.dispatcher.DispatcherGateway in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testErrorHandlerIsCalled.

private void testErrorHandlerIsCalled(Supplier<CompletableFuture<Acknowledge>> shutdownFunction) throws Exception {
    final TestingDispatcherGateway.Builder dispatcherBuilder = TestingDispatcherGateway.newBuilder().setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get())).setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED)).setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createJobResult(jobId, ApplicationStatus.SUCCEEDED))).setClusterShutdownFunction(status -> shutdownFunction.get());
    // we're "listening" on this to be completed to verify that the error handler is called.
    // In production, this will shut down the cluster with an exception.
    final CompletableFuture<Void> errorHandlerFuture = new CompletableFuture<>();
    final TestingDispatcherGateway dispatcherGateway = dispatcherBuilder.build();
    final ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(3, dispatcherGateway, scheduledExecutor, errorHandlerFuture::completeExceptionally);
    final CompletableFuture<Acknowledge> completionFuture = bootstrap.getBootstrapCompletionFuture();
    // we call the error handler
    assertException(errorHandlerFuture, FlinkRuntimeException.class);
    // we return a future that is completed exceptionally
    assertException(completionFuture, FlinkRuntimeException.class);
}

Example 15 with DispatcherGateway

use of org.apache.flink.runtime.dispatcher.DispatcherGateway in project flink by apache.

the class JobSubmitHandlerTest method testRejectionOnCountMismatch.

@Test
public void testRejectionOnCountMismatch() throws Exception {
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
        objectOut.writeObject(JobGraphTestUtils.emptyJobGraph());
    }
    final Path countExceedingFile = TEMPORARY_FOLDER.newFile().toPath();
    TestingDispatcherGateway.Builder builder = TestingDispatcherGateway.newBuilder();
    builder.setBlobServerPort(blobServer.getPort()).setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get())).setHostname("localhost");
    DispatcherGateway mockGateway = builder.build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(mockGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());
    try {
        handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance(), Arrays.asList(jobGraphFile.toFile(), countExceedingFile.toFile())), mockGateway).get();
    } catch (Exception e) {
        ExceptionUtils.findThrowable(e, candidate -> candidate instanceof RestHandlerException && candidate.getMessage().contains("count"));
    }
}