Search in sources :

Example 1 with SubmitJobRequest

use of com.google.cloud.dataproc.v1beta2.SubmitJobRequest in project cdap by caskdata.

the class DataprocRuntimeJobManager method launch.

@Override
public void launch(RuntimeJobInfo runtimeJobInfo) throws Exception {
    String bucket = DataprocUtils.getBucketName(this.bucket);
    ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo();
    LOG.debug("Launching run {} with following configurations: cluster {}, project {}, region {}, bucket {}.", runInfo.getRun(), clusterName, projectId, region, bucket);
    // TODO: CDAP-16408 use fixed directory for caching twill, application, artifact jars
    File tempDir = Files.createTempDirectory("dataproc.launcher").toFile();
    // on dataproc bucket the run root will be <bucket>/cdap-job/<runid>/. All the files for this run will be copied
    // under that base dir.
    String runRootPath = getPath(DataprocUtils.CDAP_GCS_ROOT, runInfo.getRun());
    try {
        // step 1: build twill.jar and launcher.jar and add them to files to be copied to gcs
        List<LocalFile> localFiles = getRuntimeLocalFiles(runtimeJobInfo.getLocalizeFiles(), tempDir);
        // step 2: upload all the necessary files to gcs so that those files are available to dataproc job
        List<Future<LocalFile>> uploadFutures = new ArrayList<>();
        for (LocalFile fileToUpload : localFiles) {
            String targetFilePath = getPath(runRootPath, fileToUpload.getName());
            uploadFutures.add(provisionerContext.execute(() -> uploadFile(bucket, targetFilePath, fileToUpload)).toCompletableFuture());
        }
        List<LocalFile> uploadedFiles = new ArrayList<>();
        for (Future<LocalFile> uploadFuture : uploadFutures) {
            uploadedFiles.add(uploadFuture.get());
        }
        // step 3: build the hadoop job request to be submitted to dataproc
        SubmitJobRequest request = getSubmitJobRequest(runtimeJobInfo, uploadedFiles);
        // step 4: submit hadoop job to dataproc
        try {
            Job job = getJobControllerClient().submitJob(request);
            LOG.debug("Successfully submitted hadoop job {} to cluster {}.", job.getReference().getJobId(), clusterName);
        } catch (AlreadyExistsException ex) {
            // the job id already exists, ignore the job.
            LOG.warn("The dataproc job {} already exists. Ignoring resubmission of the job.", request.getJob().getReference().getJobId());
        }
        DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count");
    } catch (Exception e) {
        // delete all uploaded gcs files in case of exception
        DataprocUtils.deleteGCSPath(getStorageClient(), bucket, runRootPath);
        DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count", e);
        throw new Exception(String.format("Error while launching job %s on cluster %s", getJobId(runInfo), clusterName), e);
    } finally {
        // delete local temp directory
        deleteDirectoryContents(tempDir);
    }
}
Also used : AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) ArrayList(java.util.ArrayList) SubmitJobRequest(com.google.cloud.dataproc.v1beta2.SubmitJobRequest) AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) IOException(java.io.IOException) ApiException(com.google.api.gax.rpc.ApiException) StorageException(com.google.cloud.storage.StorageException) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) LocalFile(org.apache.twill.api.LocalFile) Future(java.util.concurrent.Future) HadoopJob(com.google.cloud.dataproc.v1beta2.HadoopJob) Job(com.google.cloud.dataproc.v1beta2.Job) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) LocalFile(org.apache.twill.api.LocalFile) File(java.io.File) ProgramRunInfo(io.cdap.cdap.runtime.spi.ProgramRunInfo)

Example 2 with SubmitJobRequest

use of com.google.cloud.dataproc.v1beta2.SubmitJobRequest in project cdap by caskdata.

the class DataprocRuntimeJobManager method getSubmitJobRequest.

/**
 * Creates and returns dataproc job submit request.
 */
private SubmitJobRequest getSubmitJobRequest(RuntimeJobInfo runtimeJobInfo, List<LocalFile> localFiles) {
    ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo();
    String runId = runInfo.getRun();
    // The DataprocJobMain argument is <class-name> <spark-compat> <list of archive files...>
    List<String> arguments = new ArrayList<>();
    arguments.add("--" + DataprocJobMain.RUNTIME_JOB_CLASS + "=" + runtimeJobInfo.getRuntimeJobClassname());
    arguments.add("--" + DataprocJobMain.SPARK_COMPAT + "=" + provisionerContext.getSparkCompat().getCompat());
    localFiles.stream().filter(LocalFile::isArchive).map(f -> "--" + DataprocJobMain.ARCHIVE + "=" + f.getName()).forEach(arguments::add);
    for (Map.Entry<String, String> entry : runtimeJobInfo.getJvmProperties().entrySet()) {
        arguments.add("--" + DataprocJobMain.PROPERTY_PREFIX + entry.getKey() + "=\"" + entry.getValue() + "\"");
    }
    Map<String, String> properties = new LinkedHashMap<>();
    properties.put(CDAP_RUNTIME_NAMESPACE, runInfo.getNamespace());
    properties.put(CDAP_RUNTIME_APPLICATION, runInfo.getApplication());
    properties.put(CDAP_RUNTIME_VERSION, runInfo.getVersion());
    properties.put(CDAP_RUNTIME_PROGRAM, runInfo.getProgram());
    properties.put(CDAP_RUNTIME_PROGRAM_TYPE, runInfo.getProgramType());
    properties.put(CDAP_RUNTIME_RUNID, runId);
    HadoopJob.Builder hadoopJobBuilder = HadoopJob.newBuilder().setMainClass(DataprocJobMain.class.getName()).addAllArgs(arguments).putAllProperties(properties);
    for (LocalFile localFile : localFiles) {
        // add jar file
        URI uri = localFile.getURI();
        if (localFile.getName().endsWith("jar")) {
            hadoopJobBuilder.addJarFileUris(uri.toString());
        } else {
            hadoopJobBuilder.addFileUris(uri.toString());
        }
    }
    return SubmitJobRequest.newBuilder().setRegion(region).setProjectId(projectId).setJob(Job.newBuilder().setReference(JobReference.newBuilder().setJobId(getJobId(runInfo))).setPlacement(JobPlacement.newBuilder().setClusterName(clusterName).build()).putAllLabels(labels).putLabels(LABEL_CDAP_PROGRAM, runInfo.getProgram().toLowerCase()).putLabels(LABEL_CDAP_PROGRAM_TYPE, runInfo.getProgramType().toLowerCase()).setHadoopJob(hadoopJobBuilder.build()).build()).build();
}
Also used : JobStatus(com.google.cloud.dataproc.v1beta2.JobStatus) JobPlacement(com.google.cloud.dataproc.v1beta2.JobPlacement) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) LoggerFactory(org.slf4j.LoggerFactory) FixedCredentialsProvider(com.google.api.gax.core.FixedCredentialsProvider) BlobId(com.google.cloud.storage.BlobId) StorageOptions(com.google.cloud.storage.StorageOptions) JobControllerSettings(com.google.cloud.dataproc.v1beta2.JobControllerSettings) Future(java.util.concurrent.Future) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) JobReference(com.google.cloud.dataproc.v1beta2.JobReference) Map(java.util.Map) CredentialsProvider(com.google.api.gax.core.CredentialsProvider) ProgramRunInfo(io.cdap.cdap.runtime.spi.ProgramRunInfo) URI(java.net.URI) Bucket(com.google.cloud.storage.Bucket) LocalFile(org.apache.twill.api.LocalFile) GetJobRequest(com.google.cloud.dataproc.v1beta2.GetJobRequest) Collection(java.util.Collection) Set(java.util.Set) AlreadyExistsException(com.google.api.gax.rpc.AlreadyExistsException) HadoopJob(com.google.cloud.dataproc.v1beta2.HadoopJob) List(java.util.List) ByteStreams(com.google.common.io.ByteStreams) Optional(java.util.Optional) WriteChannel(com.google.cloud.WriteChannel) Storage(com.google.cloud.storage.Storage) Pattern(java.util.regex.Pattern) Joiner(com.google.common.base.Joiner) DataprocUtils(io.cdap.cdap.runtime.spi.common.DataprocUtils) ProvisionerContext(io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext) Job(com.google.cloud.dataproc.v1beta2.Job) SubmitJobRequest(com.google.cloud.dataproc.v1beta2.SubmitJobRequest) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) ImmutableList(com.google.common.collect.ImmutableList) ListJobsRequest(com.google.cloud.dataproc.v1beta2.ListJobsRequest) BlobInfo(com.google.cloud.storage.BlobInfo) Logger(org.slf4j.Logger) Files(java.nio.file.Files) GoogleCredentials(com.google.auth.oauth2.GoogleCredentials) Channels(java.nio.channels.Channels) JobControllerClient(com.google.cloud.dataproc.v1beta2.JobControllerClient) IOException(java.io.IOException) LocationFactory(org.apache.twill.filesystem.LocationFactory) ApiException(com.google.api.gax.rpc.ApiException) File(java.io.File) StorageException(com.google.cloud.storage.StorageException) VisibleForTesting(com.google.common.annotations.VisibleForTesting) StatusCode(com.google.api.gax.rpc.StatusCode) Comparator(java.util.Comparator) InputStream(java.io.InputStream) HadoopJob(com.google.cloud.dataproc.v1beta2.HadoopJob) ArrayList(java.util.ArrayList) URI(java.net.URI) LinkedHashMap(java.util.LinkedHashMap) DefaultLocalFile(org.apache.twill.internal.DefaultLocalFile) LocalFile(org.apache.twill.api.LocalFile) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) ProgramRunInfo(io.cdap.cdap.runtime.spi.ProgramRunInfo)

Aggregations

AlreadyExistsException (com.google.api.gax.rpc.AlreadyExistsException)2 ApiException (com.google.api.gax.rpc.ApiException)2 HadoopJob (com.google.cloud.dataproc.v1beta2.HadoopJob)2 Job (com.google.cloud.dataproc.v1beta2.Job)2 SubmitJobRequest (com.google.cloud.dataproc.v1beta2.SubmitJobRequest)2 StorageException (com.google.cloud.storage.StorageException)2 ProgramRunInfo (io.cdap.cdap.runtime.spi.ProgramRunInfo)2 File (java.io.File)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 CredentialsProvider (com.google.api.gax.core.CredentialsProvider)1 FixedCredentialsProvider (com.google.api.gax.core.FixedCredentialsProvider)1 StatusCode (com.google.api.gax.rpc.StatusCode)1 GoogleCredentials (com.google.auth.oauth2.GoogleCredentials)1 WriteChannel (com.google.cloud.WriteChannel)1 GetJobRequest (com.google.cloud.dataproc.v1beta2.GetJobRequest)1 JobControllerClient (com.google.cloud.dataproc.v1beta2.JobControllerClient)1 JobControllerSettings (com.google.cloud.dataproc.v1beta2.JobControllerSettings)1 JobPlacement (com.google.cloud.dataproc.v1beta2.JobPlacement)1 JobReference (com.google.cloud.dataproc.v1beta2.JobReference)1