Examples with Cluster io.cdap.cdap.runtime.spi.provisioner.Cluster used on opensource projects

Example 11 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class ProvisioningSubtask method execute.

/**
 * Executes the subtask and returns the next subtask that should be executed if there is one.
 *
 * @param taskInfo information about the task being executed, including the current cluster state
 * @return task info to be sent to the next subtask if there is one
 * @throws Exception if there was an error executing the subtask
 */
public Optional<ProvisioningTaskInfo> execute(ProvisioningTaskInfo taskInfo) throws Exception {
    Cluster cluster = taskInfo.getCluster();
    Cluster nextCluster = execute(cluster);
    return transition.apply(nextCluster).map(nextState -> {
        ProvisioningOp nextOp = new ProvisioningOp(taskInfo.getProvisioningOp().getType(), nextState);
        return new ProvisioningTaskInfo(taskInfo, nextOp, nextCluster);
    });
}

Example 12 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class ProvisioningServiceTest method testScanForTasks.

@Test
public void testScanForTasks() throws Exception {
    // write state for a provision operation that is polling for the cluster to be created
    TaskFields taskFields = createTaskInfo(new MockProvisioner.PropertyBuilder().build());
    ProvisioningOp op = new ProvisioningOp(ProvisioningOp.Type.PROVISION, ProvisioningOp.Status.POLLING_CREATE);
    Cluster cluster = new Cluster("name", ClusterStatus.CREATING, Collections.emptyList(), Collections.emptyMap());
    ProvisioningTaskInfo taskInfo = new ProvisioningTaskInfo(taskFields.programRunId, taskFields.programDescriptor, taskFields.programOptions, Collections.emptyMap(), MockProvisioner.NAME, "Bob", op, Locations.toLocation(TEMP_FOLDER.newFolder()).toURI(), cluster);
    provisionerStore.putTaskInfo(taskInfo);
    provisioningService.resumeTasks(t -> {
    });
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(taskFields.programRunId, ProvisioningOp.Type.PROVISION);
    waitForExpectedProvisioningState(taskKey, ProvisioningOp.Status.CREATED);
}

Example 13 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class ProvisioningServiceTest method testGetClusterStatusFailure.

@Test(expected = Exception.class)
public void testGetClusterStatusFailure() throws Exception {
    TaskFields taskFields = createTaskInfo(new MockProvisioner.PropertyBuilder().setFirstClusterStatus(ClusterStatus.RUNNING).failGet().setExpectedAppCDAPVersion(APP_CDAP_VERSION).build());
    Cluster cluster = new Cluster("test", ClusterStatus.NOT_EXISTS, Collections.emptyList(), Collections.emptyMap());
    provisioningService.getClusterStatus(taskFields.programRunId, taskFields.programOptions, cluster, "cdap");
}

Example 14 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class ElasticMapReduceProvisioner method createCluster.

@Override
public Cluster createCluster(ProvisionerContext context) throws Exception {
    // Generates and set the ssh key
    // or 'hadoop'
    SSHKeyPair sshKeyPair = context.getSSHContext().generate("ec2-user");
    context.getSSHContext().setSSHKeyPair(sshKeyPair);
    EMRConf conf = EMRConf.fromProvisionerContext(context);
    String clusterName = getClusterName(context.getProgramRunInfo());
    try (EMRClient client = EMRClient.fromConf(conf)) {
        // if it already exists, it means this is a retry. We can skip actually making the request
        Optional<ClusterSummary> existing = client.getUnterminatedClusterByName(clusterName);
        if (existing.isPresent()) {
            return client.getCluster(existing.get().getId()).get();
        }
        String clusterId = client.createCluster(clusterName);
        return new Cluster(clusterId, ClusterStatus.CREATING, Collections.emptyList(), Collections.emptyMap());
    }
}

Example 15 with Cluster

use of io.cdap.cdap.runtime.spi.provisioner.Cluster in project cdap by caskdata.

the class RemoteHadoopProvisioner method createCluster.

@Override
public Cluster createCluster(ProvisionerContext context) {
    RemoteHadoopConf conf = RemoteHadoopConf.fromProperties(context.getProperties());
    context.getSSHContext().setSSHKeyPair(conf.getKeyPair());
    Collection<Node> nodes = Collections.singletonList(new Node(conf.getHost(), Node.Type.MASTER, conf.getHost(), 0, Collections.emptyMap()));
    Map<String, String> properties = new HashMap<>();
    String principal = conf.getKerberosPrincipal();
    String keytab = conf.getKerberosKeytabPath();
    if (principal != null && keytab != null) {
        properties.put(ClusterProperties.KERBEROS_PRINCIPAL, principal);
        properties.put(ClusterProperties.KERBEROS_KEYTAB, keytab);
    }
    return new Cluster(conf.getHost(), ClusterStatus.RUNNING, nodes, properties);
}