Examples with RunJobFlowRequest com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest used on opensource projects

Example 6 with RunJobFlowRequest

use of com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest in project herd by FINRAOS.

the class EmrDaoImpl method createEmrCluster.

@Override
public String createEmrCluster(String clusterName, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParams) {
    RunJobFlowRequest runJobFlowRequest = getRunJobFlowRequest(clusterName, emrClusterDefinition);
    LOGGER.info("runJobFlowRequest={}", jsonHelper.objectToJson(runJobFlowRequest));
    String clusterId = emrOperations.runEmrJobFlow(getEmrClient(awsParams), runJobFlowRequest);
    LOGGER.info("EMR cluster started. emrClusterId=\"{}\"", clusterId);
    return clusterId;
}

Example 7 with RunJobFlowRequest

use of com.amazonaws.services.elasticmapreduce.model.RunJobFlowRequest in project herd by FINRAOS.

the class EmrDaoImpl method getRunJobFlowRequest.

/**
 * Create the run job flow request object.
 *
 * @param emrClusterDefinition the EMR definition name value
 * @param clusterName the EMR cluster name
 *
 * @return the run job flow request for the given configuration
 */
private RunJobFlowRequest getRunJobFlowRequest(String clusterName, EmrClusterDefinition emrClusterDefinition) {
    // Create the object
    RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition));
    // Set release label
    if (StringUtils.isNotBlank(emrClusterDefinition.getReleaseLabel())) {
        runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel());
    }
    // Set list of Applications
    List<EmrClusterDefinitionApplication> emrClusterDefinitionApplications = emrClusterDefinition.getApplications();
    if (!CollectionUtils.isEmpty(emrClusterDefinitionApplications)) {
        runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications));
    }
    // Set list of Configurations
    List<EmrClusterDefinitionConfiguration> emrClusterDefinitionConfigurations = emrClusterDefinition.getConfigurations();
    if (!CollectionUtils.isEmpty(emrClusterDefinitionConfigurations)) {
        runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations));
    }
    // Set the log bucket if specified
    if (StringUtils.isNotBlank(emrClusterDefinition.getLogBucket())) {
        runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket());
    }
    // Set the visible to all flag
    if (emrClusterDefinition.isVisibleToAll() != null) {
        runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll());
    }
    // Set the IAM profile for the nodes
    if (StringUtils.isNotBlank(emrClusterDefinition.getEc2NodeIamProfileName())) {
        runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName());
    } else {
        runJobFlowRequest.setJobFlowRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME));
    }
    // Set the IAM profile for the service
    if (StringUtils.isNotBlank(emrClusterDefinition.getServiceIamRole())) {
        runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole());
    } else {
        runJobFlowRequest.setServiceRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME));
    }
    // Set the AMI version if specified
    if (StringUtils.isNotBlank(emrClusterDefinition.getAmiVersion())) {
        runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion());
    }
    // Set the additionalInfo if specified
    if (StringUtils.isNotBlank(emrClusterDefinition.getAdditionalInfo())) {
        runJobFlowRequest.setAdditionalInfo(emrClusterDefinition.getAdditionalInfo());
    }
    // Set the bootstrap actions
    if (!getBootstrapActionConfigList(emrClusterDefinition).isEmpty()) {
        runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition));
    }
    // Set the app installation steps
    runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition));
    // Set the tags
    runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition));
    // Assign supported products as applicable
    if (StringUtils.isNotBlank(emrClusterDefinition.getSupportedProduct())) {
        List<String> supportedProducts = new ArrayList<>();
        supportedProducts.add(emrClusterDefinition.getSupportedProduct());
        runJobFlowRequest.setSupportedProducts(supportedProducts);
    }
    // Assign security configuration.
    if (StringUtils.isNotBlank(emrClusterDefinition.getSecurityConfiguration())) {
        runJobFlowRequest.setSecurityConfiguration(emrClusterDefinition.getSecurityConfiguration());
    }
    // Assign scale down behavior.
    if (StringUtils.isNotBlank(emrClusterDefinition.getScaleDownBehavior())) {
        runJobFlowRequest.setScaleDownBehavior(emrClusterDefinition.getScaleDownBehavior());
    }
    // Return the object
    return runJobFlowRequest;
}