Examples with ProvisionerContext io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext used on opensource projects

Example 6 with ProvisionerContext

use of io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext in project cdap by caskdata.

the class ExistingDataprocProvisioner method createCluster.

@Override
public Cluster createCluster(ProvisionerContext context) throws Exception {
    Map<String, String> contextProperties = createContextProperties(context);
    DataprocConf conf = DataprocConf.create(contextProperties);
    if (context.getRuntimeMonitorType() == RuntimeMonitorType.SSH) {
        String sshUser = contextProperties.get(SSH_USER);
        String sshKey = contextProperties.get(SSH_KEY);
        if (Strings.isNullOrEmpty(sshUser) || Strings.isNullOrEmpty(sshKey)) {
            throw new DataprocRuntimeException("SSH User and key are required for monitoring through SSH.");
        }
        SSHKeyPair sshKeyPair = new SSHKeyPair(new SSHPublicKey(sshUser, ""), () -> sshKey.getBytes(StandardCharsets.UTF_8));
        // The ssh context shouldn't be null, but protect it in case there is platform bug
        Optional.ofNullable(context.getSSHContext()).ifPresent(c -> c.setSSHKeyPair(sshKeyPair));
    }
    String clusterName = contextProperties.get(CLUSTER_NAME);
    try (DataprocClient client = DataprocClient.fromConf(conf, false)) {
        try {
            client.updateClusterLabels(clusterName, getSystemLabels());
        } catch (DataprocRuntimeException e) {
            // Only log the stacktrace if trace log level is enabled
            if (LOG.isTraceEnabled()) {
                LOG.trace("Cannot update cluster labels due to {}", e.getMessage(), e);
            } else {
                LOG.debug("Cannot update cluster labels due to {}", e.getMessage());
            }
        }
        return client.getCluster(clusterName).filter(c -> c.getStatus() == ClusterStatus.RUNNING).orElseThrow(() -> new DataprocRuntimeException("Dataproc cluster " + clusterName + " does not exist or not in running state."));
    }
}

Example 7 with ProvisionerContext

use of io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext in project cdap by caskdata.

the class ProvisioningService method getClusterStatus.

/**
 * Returns the {@link ClusterStatus} for the cluster being used to execute the given program run.
 *
 * @param programRunId the program run id for checking the cluster status
 * @param programOptions the program options for the given run
 * @param cluster the {@link Cluster} information for the given run
 * @param userId the user id to use for {@link SecureStore} operation.
 * @return the {@link ClusterStatus}
 * @throws Exception if non-retryable exception is encountered when querying cluster status
 */
public ClusterStatus getClusterStatus(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster, String userId) throws Exception {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // If there is no provisioner available, we can't do anything further, hence returning NOT_EXISTS
    if (provisioner == null) {
        return ClusterStatus.NOT_EXISTS;
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    // Create the ProvisionerContext and query the cluster status using the provisioner
    ProvisionerContext context;
    try {
        DefaultSSHContext defaultSSHContext = null;
        if (!getRuntimeJobManager(programRunId, programOptions).isPresent()) {
            defaultSSHContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, null);
        }
        context = createContext(cConf, programOptions, programRunId, userId, properties, defaultSSHContext);
    } catch (InvalidMacroException e) {
        // This shouldn't happen
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while checking cluster status.", e));
        return ClusterStatus.NOT_EXISTS;
    }
    return Retries.callWithRetries(() -> provisioner.getClusterStatus(context, cluster), RetryStrategies.exponentialDelay(1, 5, TimeUnit.SECONDS), RetryableProvisionException.class::isInstance);
}

Example 8 with ProvisionerContext

use of io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext in project cdap by caskdata.

the class ProvisioningService method createDeprovisionTask.

private Runnable createDeprovisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner, Consumer<ProgramRunId> taskCleanup) {
    Map<String, String> properties = taskInfo.getProvisionerProperties();
    ProvisionerContext context;
    SSHKeyPair sshKeyPair = null;
    try {
        sshKeyPair = createSSHKeyPair(taskInfo);
    } catch (IOException e) {
        LOG.warn("Failed to load ssh key. No SSH key will be available for the deprovision task", e);
    }
    ProgramRunId programRunId = taskInfo.getProgramRunId();
    Map<String, String> systemArgs = taskInfo.getProgramOptions().getArguments().asMap();
    try {
        SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, sshKeyPair);
        context = createContext(cConf, taskInfo.getProgramOptions(), programRunId, taskInfo.getUser(), properties, sshContext);
    } catch (InvalidMacroException e) {
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while deprovisoning. " + "The cluster will be marked as orphaned.", e));
        provisionerNotifier.orphaned(programRunId);
        return () -> {
        };
    }
    DeprovisionTask task = new DeprovisionTask(taskInfo, transactionRunner, 300, provisioner, context, provisionerNotifier, locationFactory);
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.DEPROVISION);
    return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
        try {
            long delay = task.executeOnce();
            if (delay < 0) {
                taskCleanup.accept(programRunId);
            }
            return delay;
        } catch (InterruptedException e) {
            // We can get interrupted if the task is cancelled or CDAP is stopped. In either case, just return.
            // If it was cancelled, state cleanup is left to the caller. If it was CDAP master stopping, the task
            // will be resumed on master startup
            LOG.debug("Deprovision task for program run {} interrupted.", programRunId);
            throw e;
        } catch (Exception e) {
            // Otherwise, if there was an error deprovisioning, run the cleanup
            LOG.info("Deprovision task for program run {} failed.", programRunId, e);
            taskCleanup.accept(programRunId);
            throw e;
        }
    }));
}

Example 9 with ProvisionerContext

use of io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext in project cdap by cdapio.

the class ProvisioningService method getRuntimeJobManager.

/**
 * Returns runtime job manager implementation.
 *
 * @param programRunId program run
 * @param programOptions program options
 * @return an object of runtime job manager
 */
public Optional<RuntimeJobManager> getRuntimeJobManager(ProgramRunId programRunId, ProgramOptions programOptions) {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    String user = programOptions.getArguments().getOption(ProgramOptionConstants.USER_ID);
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    ProvisionerContext context = createContext(cConf, programOptions, programRunId, user, properties, null);
    return provisioner.getRuntimeJobManager(context);
}

Example 10 with ProvisionerContext

use of io.cdap.cdap.runtime.spi.provisioner.ProvisionerContext in project cdap by cdapio.

the class ProvisioningService method createProvisionTask.

private Runnable createProvisionTask(ProvisioningTaskInfo taskInfo, Provisioner provisioner) {
    ProgramRunId programRunId = taskInfo.getProgramRunId();
    ProgramOptions programOptions = taskInfo.getProgramOptions();
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    ProvisionerContext context;
    try {
        SSHContext sshContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), locationFactory.create(taskInfo.getSecureKeysDir()), createSSHKeyPair(taskInfo));
        context = createContext(cConf, programOptions, programRunId, taskInfo.getUser(), taskInfo.getProvisionerProperties(), sshContext);
    } catch (IOException e) {
        runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Failed to load ssh key. The run will be marked as failed.", e));
        programStateWriter.error(programRunId, new IllegalStateException("Failed to load ssh key.", e));
        provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
        return () -> {
        };
    } catch (InvalidMacroException e) {
        runWithProgramLogging(taskInfo.getProgramRunId(), systemArgs, () -> LOG.error("Could not evaluate macros while provisoning. " + "The run will be marked as failed.", e));
        programStateWriter.error(programRunId, new IllegalStateException("Could not evaluate macros while provisioning", e));
        provisionerNotifier.deprovisioning(taskInfo.getProgramRunId());
        return () -> {
        };
    }
    // TODO: (CDAP-13246) pick up timeout from profile instead of hardcoding
    ProvisioningTask task = new ProvisionTask(taskInfo, transactionRunner, provisioner, context, provisionerNotifier, programStateWriter, 300);
    ProvisioningTaskKey taskKey = new ProvisioningTaskKey(programRunId, ProvisioningOp.Type.PROVISION);
    return () -> taskExecutor.submit(taskKey, () -> callWithProgramLogging(programRunId, systemArgs, () -> {
        try {
            return task.executeOnce();
        } catch (InterruptedException e) {
            LOG.debug("Provision task for program run {} interrupted.", taskInfo.getProgramRunId());
            throw e;
        } catch (Exception e) {
            LOG.info("Provision task for program run {} failed.", taskInfo.getProgramRunId(), e);
            throw e;
        }
    }));
}