Examples with JobSubmitRequestBody org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody used on opensource projects

Example 1 with JobSubmitRequestBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody in project flink by apache.

the class JobSubmitHandler method handleRequest.

@Override
protected CompletableFuture<JobSubmitResponseBody> handleRequest(@Nonnull HandlerRequest<JobSubmitRequestBody> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
    final Collection<File> uploadedFiles = request.getUploadedFiles();
    final Map<String, Path> nameToFile = uploadedFiles.stream().collect(Collectors.toMap(File::getName, Path::fromLocalFile));
    if (uploadedFiles.size() != nameToFile.size()) {
        throw new RestHandlerException(String.format("The number of uploaded files was %s than the expected count. Expected: %s Actual %s", uploadedFiles.size() < nameToFile.size() ? "lower" : "higher", nameToFile.size(), uploadedFiles.size()), HttpResponseStatus.BAD_REQUEST);
    }
    final JobSubmitRequestBody requestBody = request.getRequestBody();
    if (requestBody.jobGraphFileName == null) {
        throw new RestHandlerException(String.format("The %s field must not be omitted or be null.", JobSubmitRequestBody.FIELD_NAME_JOB_GRAPH), HttpResponseStatus.BAD_REQUEST);
    }
    CompletableFuture<JobGraph> jobGraphFuture = loadJobGraph(requestBody, nameToFile);
    Collection<Path> jarFiles = getJarFilesToUpload(requestBody.jarFileNames, nameToFile);
    Collection<Tuple2<String, Path>> artifacts = getArtifactFilesToUpload(requestBody.artifactFileNames, nameToFile);
    CompletableFuture<JobGraph> finalizedJobGraphFuture = uploadJobGraphFiles(gateway, jobGraphFuture, jarFiles, artifacts, configuration);
    CompletableFuture<Acknowledge> jobSubmissionFuture = finalizedJobGraphFuture.thenCompose(jobGraph -> gateway.submitJob(jobGraph, timeout));
    return jobSubmissionFuture.thenCombine(jobGraphFuture, (ack, jobGraph) -> new JobSubmitResponseBody("/jobs/" + jobGraph.getJobID()));
}

Example 2 with JobSubmitRequestBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody in project flink by apache.

the class JobSubmitHandlerTest method testFailedJobSubmission.

@Test
public void testFailedJobSubmission() throws Exception {
    final String errorMessage = "test";
    DispatcherGateway mockGateway = TestingDispatcherGateway.newBuilder().setSubmitFunction(jobgraph -> FutureUtils.completedExceptionally(new Exception(errorMessage))).build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(mockGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
        objectOut.writeObject(jobGraph);
    }
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());
    try {
        handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance(), Collections.singletonList(jobGraphFile.toFile())), mockGateway).get();
    } catch (Exception e) {
        Throwable t = ExceptionUtils.stripExecutionException(e);
        Assert.assertEquals(errorMessage, t.getMessage());
    }
}

Example 3 with JobSubmitRequestBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody in project flink by apache.

the class JobSubmitHandlerTest method testFileHandling.

@Test
public void testFileHandling() throws Exception {
    final String dcEntryName = "entry";
    CompletableFuture<JobGraph> submittedJobGraphFuture = new CompletableFuture<>();
    DispatcherGateway dispatcherGateway = TestingDispatcherGateway.newBuilder().setBlobServerPort(blobServer.getPort()).setSubmitFunction(submittedJobGraph -> {
        submittedJobGraphFuture.complete(submittedJobGraph);
        return CompletableFuture.completedFuture(Acknowledge.get());
    }).build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(dispatcherGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    final Path jarFile = TEMPORARY_FOLDER.newFile().toPath();
    final Path artifactFile = TEMPORARY_FOLDER.newFile().toPath();
    final JobGraph jobGraph = JobGraphTestUtils.emptyJobGraph();
    // the entry that should be updated
    jobGraph.addUserArtifact(dcEntryName, new DistributedCache.DistributedCacheEntry("random", false));
    try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
        objectOut.writeObject(jobGraph);
    }
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.singletonList(jarFile.getFileName().toString()), Collections.singleton(new JobSubmitRequestBody.DistributedCacheFile(dcEntryName, artifactFile.getFileName().toString())));
    handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance(), Arrays.asList(jobGraphFile.toFile(), jarFile.toFile(), artifactFile.toFile())), dispatcherGateway).get();
    Assert.assertTrue("No JobGraph was submitted.", submittedJobGraphFuture.isDone());
    final JobGraph submittedJobGraph = submittedJobGraphFuture.get();
    Assert.assertEquals(1, submittedJobGraph.getUserJarBlobKeys().size());
    Assert.assertEquals(1, submittedJobGraph.getUserArtifacts().size());
    Assert.assertNotNull(submittedJobGraph.getUserArtifacts().get(dcEntryName).blobKey);
}

Example 4 with JobSubmitRequestBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody in project flink by apache.

the class JobSubmitHandlerTest method testRejectionOnCountMismatch.

@Test
public void testRejectionOnCountMismatch() throws Exception {
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
        objectOut.writeObject(JobGraphTestUtils.emptyJobGraph());
    }
    final Path countExceedingFile = TEMPORARY_FOLDER.newFile().toPath();
    TestingDispatcherGateway.Builder builder = TestingDispatcherGateway.newBuilder();
    builder.setBlobServerPort(blobServer.getPort()).setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get())).setHostname("localhost");
    DispatcherGateway mockGateway = builder.build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(mockGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());
    try {
        handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance(), Arrays.asList(jobGraphFile.toFile(), countExceedingFile.toFile())), mockGateway).get();
    } catch (Exception e) {
        ExceptionUtils.findThrowable(e, candidate -> candidate instanceof RestHandlerException && candidate.getMessage().contains("count"));
    }
}

Example 5 with JobSubmitRequestBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitRequestBody in project flink by apache.

the class JobSubmitHandlerTest method testSerializationFailureHandling.

@Test
public void testSerializationFailureHandling() throws Exception {
    final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
    DispatcherGateway mockGateway = TestingDispatcherGateway.newBuilder().setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get())).build();
    JobSubmitHandler handler = new JobSubmitHandler(() -> CompletableFuture.completedFuture(mockGateway), RpcUtils.INF_TIMEOUT, Collections.emptyMap(), TestingUtils.defaultExecutor(), configuration);
    JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());
    try {
        handler.handleRequest(HandlerRequest.create(request, EmptyMessageParameters.getInstance()), mockGateway);
        Assert.fail();
    } catch (RestHandlerException rhe) {
        Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
    }
}