Examples with ProgramRunInfo io.cdap.cdap.runtime.spi.ProgramRunInfo used on opensource projects

Example 1 with ProgramRunInfo

use of io.cdap.cdap.runtime.spi.ProgramRunInfo in project cdap by caskdata.

the class TetheringRuntimeJobManager method launch.

@Override
public void launch(RuntimeJobInfo runtimeJobInfo) throws Exception {
    ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo();
    LOG.debug("Launching run {} with following configurations: tethered instance name {}, tethered namespace {}.", runInfo.getRun(), tetheredInstanceName, tetheredNamespace);
    byte[] payload = Bytes.toBytes(GSON.toJson(createLaunchPayload(runtimeJobInfo.getLocalizeFiles())));
    TetheringControlMessage message = new TetheringControlMessage(TetheringControlMessage.Type.RUN_PIPELINE, payload);
    publishToControlChannel(message);
}

Example 2 with ProgramRunInfo

use of io.cdap.cdap.runtime.spi.ProgramRunInfo in project cdap by caskdata.

the class DataprocProvisioner method getRunKey.

/**
 * Gets the identifier to locate cluster for the given context.
 * It's used as a cluster name when reuse is disabled and put into {@link #LABEL_RUN_KEY} when reuse is enabled.
 * Name must start with a lowercase letter followed by up to 51 lowercase letters,
 * numbers, or hyphens, and cannot end with a hyphen
 * We'll use app-runid, where app is truncated to fit, lowercased, and stripped of invalid characters.
 *
 * @param context the provisioner context
 * @return a string that is a valid cluster name
 */
@VisibleForTesting
String getRunKey(ProvisionerContext context) {
    ProgramRunInfo programRunInfo = context.getProgramRunInfo();
    String cleanedAppName = programRunInfo.getApplication().replaceAll("[^A-Za-z0-9\\-]", "").toLowerCase();
    // 51 is max length, need to subtract the prefix and 1 extra for the '-' separating app name and run id
    int maxAppLength = 51 - CLUSTER_PREFIX.length() - 1 - programRunInfo.getRun().length();
    if (cleanedAppName.length() > maxAppLength) {
        cleanedAppName = cleanedAppName.substring(0, maxAppLength);
    }
    return CLUSTER_PREFIX + cleanedAppName + "-" + programRunInfo.getRun();
}

Example 3 with ProgramRunInfo

use of io.cdap.cdap.runtime.spi.ProgramRunInfo in project cdap by caskdata.

the class DataprocProvisionerTest method testClusterCreateNoReuse.

@Test
public void testClusterCreateNoReuse() throws Exception {
    context.addProperty("accountKey", "testKey");
    context.addProperty(DataprocConf.PROJECT_ID_KEY, "testProject");
    context.addProperty("region", "testRegion");
    context.addProperty("idleTTL", "5");
    context.addProperty(DataprocConf.SKIP_DELETE, "true");
    context.setProfileName("testProfile");
    ProgramRunInfo programRunInfo = new ProgramRunInfo.Builder().setNamespace("ns").setApplication("app").setVersion("1.0").setProgramType("workflow").setProgram("program").setRun("runId").build();
    context.setProgramRunInfo(programRunInfo);
    context.setSparkCompat(SparkCompat.SPARK2_2_11);
    context.addProperty(DataprocConf.CLUSTER_REUSE_ENABLED, "false");
    Mockito.when(dataprocClient.getCluster("cdap-app-runId")).thenReturn(Optional.empty());
    Mockito.when(dataprocClient.createCluster(Mockito.eq("cdap-app-runId"), Mockito.eq("1.3"), addedLabelsCaptor.capture(), Mockito.eq(false))).thenReturn(ClusterOperationMetadata.getDefaultInstance());
    Cluster expectedCluster = new Cluster("cdap-app-runId", ClusterStatus.CREATING, Collections.emptyList(), Collections.emptyMap());
    Assert.assertEquals(expectedCluster, provisioner.createCluster(context));
    Assert.assertEquals(Collections.singletonMap("cdap-version", "6_4"), addedLabelsCaptor.getValue());
}

Example 4 with ProgramRunInfo

use of io.cdap.cdap.runtime.spi.ProgramRunInfo in project cdap by caskdata.

the class DataprocRuntimeJobManagerTest method jobNameTest.

@Test
public void jobNameTest() {
    ProgramRunInfo runInfo = new ProgramRunInfo.Builder().setNamespace("namespace").setApplication("application").setVersion("1.0").setProgramType("workflow").setProgram("program").setRun(UUID.randomUUID().toString()).build();
    String jobName = DataprocRuntimeJobManager.getJobId(runInfo);
    Assert.assertTrue(jobName.startsWith("namespace_application_program"));
}

Example 5 with ProgramRunInfo

use of io.cdap.cdap.runtime.spi.ProgramRunInfo in project cdap by caskdata.

the class DataprocRuntimeJobManager method launch.

@Override
public void launch(RuntimeJobInfo runtimeJobInfo) throws Exception {
    String bucket = DataprocUtils.getBucketName(this.bucket);
    ProgramRunInfo runInfo = runtimeJobInfo.getProgramRunInfo();
    LOG.debug("Launching run {} with following configurations: cluster {}, project {}, region {}, bucket {}.", runInfo.getRun(), clusterName, projectId, region, bucket);
    // TODO: CDAP-16408 use fixed directory for caching twill, application, artifact jars
    File tempDir = Files.createTempDirectory("dataproc.launcher").toFile();
    // on dataproc bucket the run root will be <bucket>/cdap-job/<runid>/. All the files for this run will be copied
    // under that base dir.
    String runRootPath = getPath(DataprocUtils.CDAP_GCS_ROOT, runInfo.getRun());
    try {
        // step 1: build twill.jar and launcher.jar and add them to files to be copied to gcs
        List<LocalFile> localFiles = getRuntimeLocalFiles(runtimeJobInfo.getLocalizeFiles(), tempDir);
        // step 2: upload all the necessary files to gcs so that those files are available to dataproc job
        List<Future<LocalFile>> uploadFutures = new ArrayList<>();
        for (LocalFile fileToUpload : localFiles) {
            String targetFilePath = getPath(runRootPath, fileToUpload.getName());
            uploadFutures.add(provisionerContext.execute(() -> uploadFile(bucket, targetFilePath, fileToUpload)).toCompletableFuture());
        }
        List<LocalFile> uploadedFiles = new ArrayList<>();
        for (Future<LocalFile> uploadFuture : uploadFutures) {
            uploadedFiles.add(uploadFuture.get());
        }
        // step 3: build the hadoop job request to be submitted to dataproc
        SubmitJobRequest request = getSubmitJobRequest(runtimeJobInfo, uploadedFiles);
        // step 4: submit hadoop job to dataproc
        try {
            Job job = getJobControllerClient().submitJob(request);
            LOG.debug("Successfully submitted hadoop job {} to cluster {}.", job.getReference().getJobId(), clusterName);
        } catch (AlreadyExistsException ex) {
            // the job id already exists, ignore the job.
            LOG.warn("The dataproc job {} already exists. Ignoring resubmission of the job.", request.getJob().getReference().getJobId());
        }
        DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count");
    } catch (Exception e) {
        // delete all uploaded gcs files in case of exception
        DataprocUtils.deleteGCSPath(getStorageClient(), bucket, runRootPath);
        DataprocUtils.emitMetric(provisionerContext, region, "provisioner.submitJob.response.count", e);
        throw new Exception(String.format("Error while launching job %s on cluster %s", getJobId(runInfo), clusterName), e);
    } finally {
        // delete local temp directory
        deleteDirectoryContents(tempDir);
    }
}