Examples with Provisioner io.cdap.cdap.runtime.spi.provisioner.Provisioner used on opensource projects

Example 1 with Provisioner

use of io.cdap.cdap.runtime.spi.provisioner.Provisioner in project cdap by caskdata.

the class ProvisioningService method provision.

/**
 * Record that a cluster will be provisioned for a program run, returning a Runnable that will actually perform
 * the cluster provisioning. This method must be run within a transaction.
 * The task returned should only be executed after the transaction that ran this method has completed.
 * Running the returned Runnable will start the actual task using an executor within this service so that it can be
 * tracked and optionally cancelled using {@link #cancelProvisionTask(ProgramRunId)}. The caller does not need to
 * submit the runnable using their own executor.
 *
 * @param provisionRequest the provision request
 * @param context context for the transaction
 * @return runnable that will actually execute the cluster provisioning
 */
public Runnable provision(ProvisionRequest provisionRequest, StructuredTableContext context) throws IOException {
    ProgramRunId programRunId = provisionRequest.getProgramRunId();
    ProgramOptions programOptions = provisionRequest.getProgramOptions();
    Map<String, String> args = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(args);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // any errors seen here will transition the state straight to deprovisioned since no cluster create was attempted
    if (provisioner == null) {
        runWithProgramLogging(programRunId, args, () -> LOG.error("Could not provision cluster for the run because provisioner {} does not exist.", name));
        programStateWriter.error(programRunId, new IllegalStateException("Provisioner does not exist."));
        provisionerNotifier.deprovisioned(programRunId);
        return () -> {
        };
    }
    // get plugin requirement information and check for capability to run on the provisioner
    Set<PluginRequirement> requirements = GSON.fromJson(args.get(ProgramOptionConstants.PLUGIN_REQUIREMENTS), PLUGIN_REQUIREMENT_SET_TYPE);
    if (requirements != null) {
        Set<PluginRequirement> unfulfilledRequirements = getUnfulfilledRequirements(provisioner.getCapabilities(), requirements);
        if (!unfulfilledRequirements.isEmpty()) {
            runWithProgramLogging(programRunId, args, () -> LOG.error(String.format("'%s' cannot be run using profile '%s' because the profile does not met all " + "plugin requirements. Following requirements were not meet by the listed " + "plugins: '%s'", programRunId.getProgram(), name, groupByRequirement(unfulfilledRequirements))));
            programStateWriter.error(programRunId, new IllegalArgumentException("Provisioner does not meet all the " + "requirements for the program to run."));
            provisionerNotifier.deprovisioned(programRunId);
            return () -> {
            };
        }
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(args);
    ProvisioningOp provisioningOp = new ProvisioningOp(ProvisioningOp.Type.PROVISION, ProvisioningOp.Status.REQUESTING_CREATE);
    ProvisioningTaskInfo provisioningTaskInfo = new ProvisioningTaskInfo(programRunId, provisionRequest.getProgramDescriptor(), programOptions, properties, name, provisionRequest.getUser(), provisioningOp, createKeysDirectory(programRunId).toURI(), null);
    ProvisionerTable provisionerTable = new ProvisionerTable(context);
    provisionerTable.putTaskInfo(provisioningTaskInfo);
    return createProvisionTask(provisioningTaskInfo, provisioner);
}

Example 2 with Provisioner

use of io.cdap.cdap.runtime.spi.provisioner.Provisioner in project cdap by caskdata.

the class ProvisioningService method getRuntimeJobManager.

/**
 * Returns runtime job manager implementation.
 *
 * @param programRunId program run
 * @param programOptions program options
 * @return an object of runtime job manager
 */
public Optional<RuntimeJobManager> getRuntimeJobManager(ProgramRunId programRunId, ProgramOptions programOptions) {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    String user = programOptions.getArguments().getOption(ProgramOptionConstants.USER_ID);
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    ProvisionerContext context = createContext(cConf, programOptions, programRunId, user, properties, null);
    return provisioner.getRuntimeJobManager(context);
}

Example 3 with Provisioner

use of io.cdap.cdap.runtime.spi.provisioner.Provisioner in project cdap by caskdata.

the class ProvisioningService method validateProperties.

/**
 * Validate properties for the specified provisioner.
 *
 * @param provisionerName the name of the provisioner to validate
 * @param properties properties for the specified provisioner
 * @throws NotFoundException if the provisioner does not exist
 * @throws IllegalArgumentException if the properties are invalid
 */
public void validateProperties(String provisionerName, Map<String, String> properties) throws NotFoundException {
    Provisioner provisioner = provisionerInfo.get().provisioners.get(provisionerName);
    if (provisioner == null) {
        throw new NotFoundException(String.format("Provisioner '%s' does not exist", provisionerName));
    }
    provisioner.validateProperties(properties);
}

Example 4 with Provisioner

use of io.cdap.cdap.runtime.spi.provisioner.Provisioner in project cdap by caskdata.

the class ProvisioningService method getClusterStatus.

/**
 * Returns the {@link ClusterStatus} for the cluster being used to execute the given program run.
 *
 * @param programRunId the program run id for checking the cluster status
 * @param programOptions the program options for the given run
 * @param cluster the {@link Cluster} information for the given run
 * @param userId the user id to use for {@link SecureStore} operation.
 * @return the {@link ClusterStatus}
 * @throws Exception if non-retryable exception is encountered when querying cluster status
 */
public ClusterStatus getClusterStatus(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster, String userId) throws Exception {
    Map<String, String> systemArgs = programOptions.getArguments().asMap();
    String name = SystemArguments.getProfileProvisioner(systemArgs);
    Provisioner provisioner = provisionerInfo.get().provisioners.get(name);
    // If there is no provisioner available, we can't do anything further, hence returning NOT_EXISTS
    if (provisioner == null) {
        return ClusterStatus.NOT_EXISTS;
    }
    Map<String, String> properties = SystemArguments.getProfileProperties(systemArgs);
    // Create the ProvisionerContext and query the cluster status using the provisioner
    ProvisionerContext context;
    try {
        DefaultSSHContext defaultSSHContext = null;
        if (!getRuntimeJobManager(programRunId, programOptions).isPresent()) {
            defaultSSHContext = new DefaultSSHContext(Networks.getAddress(cConf, Constants.NETWORK_PROXY_ADDRESS), null, null);
        }
        context = createContext(cConf, programOptions, programRunId, userId, properties, defaultSSHContext);
    } catch (InvalidMacroException e) {
        // This shouldn't happen
        runWithProgramLogging(programRunId, systemArgs, () -> LOG.error("Could not evaluate macros while checking cluster status.", e));
        return ClusterStatus.NOT_EXISTS;
    }
    return Retries.callWithRetries(() -> provisioner.getClusterStatus(context, cluster), RetryStrategies.exponentialDelay(1, 5, TimeUnit.SECONDS), RetryableProvisionException.class::isInstance);
}

Example 5 with Provisioner

use of io.cdap.cdap.runtime.spi.provisioner.Provisioner in project cdap by caskdata.

the class ProvisioningService method initializeProvisioners.

/**
 * Reloads provisioners in the extension directory. Any new provisioners will be added and any deleted provisioners
 * will be removed. Loaded provisioners will be initialized.
 */
private void initializeProvisioners() {
    Map<String, Provisioner> provisioners = provisionerProvider.loadProvisioners();
    Map<String, ProvisionerConfig> provisionerConfigs = provisionerConfigProvider.loadProvisionerConfigs(provisioners.values());
    LOG.debug("Provisioners = {}", provisioners);
    Map<String, ProvisionerDetail> details = new HashMap<>(provisioners.size());
    for (Map.Entry<String, Provisioner> provisionerEntry : provisioners.entrySet()) {
        String provisionerName = provisionerEntry.getKey();
        Provisioner provisioner = provisionerEntry.getValue();
        ProvisionerSystemContext provisionerSystemContext = new DefaultSystemProvisionerContext(cConf, provisionerName);
        try {
            provisioner.initialize(provisionerSystemContext);
        } catch (RuntimeException e) {
            LOG.warn("Error initializing the {} provisioner. It will not be available for use.", provisionerName, e);
            provisioners.remove(provisionerName);
            continue;
        }
        ProvisionerSpecification spec = provisioner.getSpec();
        ProvisionerConfig config = provisionerConfigs.getOrDefault(provisionerName, new ProvisionerConfig(new ArrayList<>(), null, null, false));
        details.put(provisionerName, new ProvisionerDetail(spec.getName(), spec.getLabel(), spec.getDescription(), config.getConfigurationGroups(), config.getFilters(), config.getIcon(), config.isBeta()));
    }
    provisionerInfo.set(new ProvisionerInfo(provisioners, details));
}