Examples with JobSubmitResponseBody org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody used on opensource projects

Example 1 with JobSubmitResponseBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody in project flink by apache.

the class JobSubmitHandler method handleRequest.

@Override
protected CompletableFuture<JobSubmitResponseBody> handleRequest(@Nonnull HandlerRequest<JobSubmitRequestBody> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
    final Collection<File> uploadedFiles = request.getUploadedFiles();
    final Map<String, Path> nameToFile = uploadedFiles.stream().collect(Collectors.toMap(File::getName, Path::fromLocalFile));
    if (uploadedFiles.size() != nameToFile.size()) {
        throw new RestHandlerException(String.format("The number of uploaded files was %s than the expected count. Expected: %s Actual %s", uploadedFiles.size() < nameToFile.size() ? "lower" : "higher", nameToFile.size(), uploadedFiles.size()), HttpResponseStatus.BAD_REQUEST);
    }
    final JobSubmitRequestBody requestBody = request.getRequestBody();
    if (requestBody.jobGraphFileName == null) {
        throw new RestHandlerException(String.format("The %s field must not be omitted or be null.", JobSubmitRequestBody.FIELD_NAME_JOB_GRAPH), HttpResponseStatus.BAD_REQUEST);
    }
    CompletableFuture<JobGraph> jobGraphFuture = loadJobGraph(requestBody, nameToFile);
    Collection<Path> jarFiles = getJarFilesToUpload(requestBody.jarFileNames, nameToFile);
    Collection<Tuple2<String, Path>> artifacts = getArtifactFilesToUpload(requestBody.artifactFileNames, nameToFile);
    CompletableFuture<JobGraph> finalizedJobGraphFuture = uploadJobGraphFiles(gateway, jobGraphFuture, jarFiles, artifacts, configuration);
    CompletableFuture<Acknowledge> jobSubmissionFuture = finalizedJobGraphFuture.thenCompose(jobGraph -> gateway.submitJob(jobGraph, timeout));
    return jobSubmissionFuture.thenCombine(jobGraphFuture, (ack, jobGraph) -> new JobSubmitResponseBody("/jobs/" + jobGraph.getJobID()));
}

Example 2 with JobSubmitResponseBody

use of org.apache.flink.runtime.rest.messages.job.JobSubmitResponseBody in project flink by apache.

the class RestClusterClient method submitJob.

@Override
public CompletableFuture<JobID> submitJob(@Nonnull JobGraph jobGraph) {
    CompletableFuture<java.nio.file.Path> jobGraphFileFuture = CompletableFuture.supplyAsync(() -> {
        try {
            final java.nio.file.Path jobGraphFile = Files.createTempFile("flink-jobgraph", ".bin");
            try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
                objectOut.writeObject(jobGraph);
            }
            return jobGraphFile;
        } catch (IOException e) {
            throw new CompletionException(new FlinkException("Failed to serialize JobGraph.", e));
        }
    }, executorService);
    CompletableFuture<Tuple2<JobSubmitRequestBody, Collection<FileUpload>>> requestFuture = jobGraphFileFuture.thenApply(jobGraphFile -> {
        List<String> jarFileNames = new ArrayList<>(8);
        List<JobSubmitRequestBody.DistributedCacheFile> artifactFileNames = new ArrayList<>(8);
        Collection<FileUpload> filesToUpload = new ArrayList<>(8);
        filesToUpload.add(new FileUpload(jobGraphFile, RestConstants.CONTENT_TYPE_BINARY));
        for (Path jar : jobGraph.getUserJars()) {
            jarFileNames.add(jar.getName());
            filesToUpload.add(new FileUpload(Paths.get(jar.toUri()), RestConstants.CONTENT_TYPE_JAR));
        }
        for (Map.Entry<String, DistributedCache.DistributedCacheEntry> artifacts : jobGraph.getUserArtifacts().entrySet()) {
            final Path artifactFilePath = new Path(artifacts.getValue().filePath);
            try {
                // Only local artifacts need to be uploaded.
                if (!artifactFilePath.getFileSystem().isDistributedFS()) {
                    artifactFileNames.add(new JobSubmitRequestBody.DistributedCacheFile(artifacts.getKey(), artifactFilePath.getName()));
                    filesToUpload.add(new FileUpload(Paths.get(artifactFilePath.getPath()), RestConstants.CONTENT_TYPE_BINARY));
                }
            } catch (IOException e) {
                throw new CompletionException(new FlinkException("Failed to get the FileSystem of artifact " + artifactFilePath + ".", e));
            }
        }
        final JobSubmitRequestBody requestBody = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), jarFileNames, artifactFileNames);
        return Tuple2.of(requestBody, Collections.unmodifiableCollection(filesToUpload));
    });
    final CompletableFuture<JobSubmitResponseBody> submissionFuture = requestFuture.thenCompose(requestAndFileUploads -> {
        LOG.info("Submitting job '{}' ({}).", jobGraph.getName(), jobGraph.getJobID());
        return sendRetriableRequest(JobSubmitHeaders.getInstance(), EmptyMessageParameters.getInstance(), requestAndFileUploads.f0, requestAndFileUploads.f1, isConnectionProblemOrServiceUnavailable(), (receiver, error) -> {
            if (error != null) {
                LOG.warn("Attempt to submit job '{}' ({}) to '{}' has failed.", jobGraph.getName(), jobGraph.getJobID(), receiver, error);
            } else {
                LOG.info("Successfully submitted job '{}' ({}) to '{}'.", jobGraph.getName(), jobGraph.getJobID(), receiver);
            }
        });
    });
    submissionFuture.thenCompose(ignored -> jobGraphFileFuture).thenAccept(jobGraphFile -> {
        try {
            Files.delete(jobGraphFile);
        } catch (IOException e) {
            LOG.warn("Could not delete temporary file {}.", jobGraphFile, e);
        }
    });
    return submissionFuture.thenApply(ignore -> jobGraph.getJobID()).exceptionally((Throwable throwable) -> {
        throw new CompletionException(new JobSubmissionException(jobGraph.getJobID(), "Failed to submit JobGraph.", ExceptionUtils.stripCompletionException(throwable)));
    });
}