Examples with Cluster io.cdap.cdap.runtime.spi.provisioner.Cluster used on opensource projects

Example 6 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class DataprocProvisionerTest method testClusterCreateNoReuse.

@Test
public void testClusterCreateNoReuse() throws Exception {
    context.addProperty("accountKey", "testKey");
    context.addProperty(DataprocConf.PROJECT_ID_KEY, "testProject");
    context.addProperty("region", "testRegion");
    context.addProperty("idleTTL", "5");
    context.addProperty(DataprocConf.SKIP_DELETE, "true");
    context.setProfileName("testProfile");
    ProgramRunInfo programRunInfo = new ProgramRunInfo.Builder().setNamespace("ns").setApplication("app").setVersion("1.0").setProgramType("workflow").setProgram("program").setRun("runId").build();
    context.setProgramRunInfo(programRunInfo);
    context.setSparkCompat(SparkCompat.SPARK2_2_11);
    context.addProperty(DataprocConf.CLUSTER_REUSE_ENABLED, "false");
    Mockito.when(dataprocClient.getCluster("cdap-app-runId")).thenReturn(Optional.empty());
    Mockito.when(dataprocClient.createCluster(Mockito.eq("cdap-app-runId"), Mockito.eq("1.3"), addedLabelsCaptor.capture(), Mockito.eq(false))).thenReturn(ClusterOperationMetadata.getDefaultInstance());
    Cluster expectedCluster = new Cluster("cdap-app-runId", ClusterStatus.CREATING, Collections.emptyList(), Collections.emptyMap());
    Assert.assertEquals(expectedCluster, provisioner.createCluster(context));
    Assert.assertEquals(Collections.singletonMap("cdap-version", "6_4"), addedLabelsCaptor.getValue());
}

Example 7 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by cdapio.

the class ProvisioningServiceTest method testGetClusterStatusFailure.

@Test(expected = Exception.class)
public void testGetClusterStatusFailure() throws Exception {
    TaskFields taskFields = createTaskInfo(new MockProvisioner.PropertyBuilder().setFirstClusterStatus(ClusterStatus.RUNNING).failGet().setExpectedAppCDAPVersion(APP_CDAP_VERSION).build());
    Cluster cluster = new Cluster("test", ClusterStatus.NOT_EXISTS, Collections.emptyList(), Collections.emptyMap());
    provisioningService.getClusterStatus(taskFields.programRunId, taskFields.programOptions, cluster, "cdap");
}

Example 8 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by cdapio.

the class SSHRemoteProcessController method isRunning.

@Override
public boolean isRunning() throws Exception {
    // Try to SSH into the host and see if the CDAP runtime process is running or not
    try (SSHSession session = new DefaultSSHSession(sshConfig)) {
        SSHProcess process = session.execute("pgrep -f -- -Dcdap.runid=" + programRunId.getRun());
        // Reading will be blocked until the process finished.
        // The output is not needed, just read it to avoid filling up the network buffer.
        ByteStreams.toByteArray(process.getInputStream());
        ByteStreams.toByteArray(process.getErrorStream());
        int exitCode = process.waitFor();
        if (exitCode != 0) {
            LOG.info("Received exit code {} when checking for remote process for program run {}.", exitCode, programRunId);
        }
        return exitCode == 0;
    } catch (IOException e) {
        // If there is error performing SSH, check if the cluster still exist and running
        LOG.debug("Failed to use SSH to determine if the remote process is running for {}. Check cluster status instead.", programRunId, e);
        Cluster cluster = GSON.fromJson(programOpts.getArguments().getOption(ProgramOptionConstants.CLUSTER), Cluster.class);
        String userId = programOpts.getArguments().getOption(ProgramOptionConstants.USER_ID);
        ClusterStatus clusterStatus = provisioningService.getClusterStatus(programRunId, programOpts, cluster, userId);
        // The cluster status has to be RUNNING in order for the remote process still has a chance that is running
        return clusterStatus == ClusterStatus.RUNNING;
    }
}

Example 9 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by cdapio.

the class ClusterInitializeSubtask method execute.

@Override
public Cluster execute(Cluster cluster) throws Exception {
    // get the full details, since many times, information like ip addresses is not available until we're done
    // polling for status and are ready to initialize. Up until now, the cluster object is what we got from
    // the original createCluster() call, except with the status updated.
    Cluster fullClusterDetails = provisioner.getClusterDetail(provisionerContext, cluster);
    provisioner.initializeCluster(provisionerContext, fullClusterDetails);
    Map<String, String> properties = new HashMap<>(cluster.getProperties());
    properties.putAll(fullClusterDetails.getProperties());
    return new Cluster(fullClusterDetails.getName(), ClusterStatus.RUNNING, fullClusterDetails.getNodes(), properties);
}

Example 10 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by cdapio.

the class DataprocProvisioner method getClusterDetail.

@Override
public Cluster getClusterDetail(ProvisionerContext context, Cluster cluster) throws Exception {
    DataprocConf conf = DataprocConf.create(createContextProperties(context));
    String clusterName = cluster.getName();
    try (DataprocClient client = getClient(conf)) {
        Optional<Cluster> existing = client.getCluster(clusterName);
        DataprocUtils.emitMetric(context, conf.getRegion(), "provisioner.clusterDetail.response.count");
        return existing.orElseGet(() -> new Cluster(cluster, ClusterStatus.NOT_EXISTS));
    } catch (Exception e) {
        DataprocUtils.emitMetric(context, conf.getRegion(), "provisioner.clusterDetail.response.count", e);
        throw e;
    }
}