Examples with EmrCluster org.finra.herd.model.api.xml.EmrCluster used on opensource projects

Example 36 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrHelperTest method testEmrAddStepsAllTypesNegativeTestCase.

/**
 * This method tests the negative test cases scenario by testing all the step types
 */
@Test
public void testEmrAddStepsAllTypesNegativeTestCase() throws Exception {
    // Create a namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create an EMR cluster definition entity.
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create EC2 on-demand pricing entities.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    EmrStepHelper stepHelper;
    // Shell script arguments
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    List<Object> steps = new ArrayList<>();
    // Shell step parameters
    EmrShellStep shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStep.setStepName("Test Shell Script");
    shellStep.setScriptArguments(shellScriptArgs);
    shellStep.setContinueOnError(true);
    steps.add(shellStep);
    // Hive step parameters
    EmrHiveStep hiveStep = new EmrHiveStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    hiveStep.setStepName("Test Hive");
    hiveStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStep.setContinueOnError(true);
    steps.add(hiveStep);
    // Pig step parameter
    EmrPigStep pigStep = new EmrPigStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    pigStep.setStepName("Test Pig");
    pigStep.setContinueOnError(true);
    pigStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    steps.add(pigStep);
    // Oozie step that includes a shell script to install oozie
    shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStep.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStep.setScriptArguments(shellScriptArgsOozie);
    steps.add(shellStep);
    // Hadoop jar step configuration
    EmrHadoopJarStep hadoopJarStep = new EmrHadoopJarStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null, null);
    hadoopJarStep.setContinueOnError(true);
    hadoopJarStep.setStepName("Hadoop Jar");
    hadoopJarStep.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStep.setMainClass("wordcount");
    steps.add(hadoopJarStep);
    for (Object emrStep : steps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStep.getClass().getName());
        emrDao.addEmrStep(emrCluster.getId(), stepHelper.getEmrStepConfig(emrStep), emrHelper.getAwsParamsDto());
    }
}

Example 37 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceImpl method createEmrClusterFromRequest.

/**
 * Creates a new EMR cluster object from request.
 *
 * @param clusterId the cluster Id.
 * @param namespaceCd the namespace Code
 * @param clusterDefinitionName the cluster definition
 * @param clusterName the cluster name
 * @param accountId the optional AWS account that EMR cluster is running in
 * @param clusterStatus the cluster status
 * @param emrClusterCreated whether EMR cluster was created.
 * @param dryRun The dry run flag.
 * @param emrClusterDefinition the EMR cluster definition.
 *
 * @return the created EMR cluster object
 */
private EmrCluster createEmrClusterFromRequest(String clusterId, String namespaceCd, String clusterDefinitionName, String clusterName, String accountId, String clusterStatus, Boolean emrClusterCreated, Boolean dryRun, EmrClusterDefinition emrClusterDefinition) {
    // Create the EMR cluster.
    EmrCluster emrCluster = new EmrCluster();
    emrCluster.setId(clusterId);
    emrCluster.setNamespace(namespaceCd);
    emrCluster.setEmrClusterDefinitionName(clusterDefinitionName);
    emrCluster.setEmrClusterName(clusterName);
    emrCluster.setAccountId(accountId);
    emrCluster.setStatus(clusterStatus);
    emrCluster.setDryRun(dryRun);
    emrCluster.setEmrClusterCreated(emrClusterCreated);
    emrCluster.setEmrClusterDefinition(emrClusterDefinition);
    return emrCluster;
}

Example 38 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceImpl method getClusterImpl.

/**
 * Gets details of an existing EMR Cluster.
 *
 * @param emrClusterAlternateKeyDto the EMR cluster alternate key
 * @param emrClusterId the cluster id of the cluster to get details
 * @param emrStepId the step id of the step to get details
 * @param verbose parameter for whether to return detailed information
 * @param accountId the optional AWS account that EMR cluster is running in
 * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets
 *
 * @return the EMR Cluster object with details.
 * @throws Exception if an error occurred while getting the cluster
 */
protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, String accountId, Boolean retrieveInstanceFleets) throws Exception {
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(accountId);
    // Perform the request validation.
    validateEmrClusterKey(emrClusterAlternateKeyDto);
    // Get the namespace and ensure it exists.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace());
    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName());
    EmrCluster emrCluster = createEmrClusterFromRequest(null, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null);
    String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName());
    try {
        // Get Cluster status if clusterId is specified
        if (StringUtils.isNotBlank(emrClusterId)) {
            Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto);
            // Validate that, Cluster exists
            Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\".");
            // Validate that, Cluster name match as specified
            Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified.");
            emrCluster.setId(cluster.getId());
            setEmrClusterStatus(emrCluster, cluster.getStatus());
        } else {
            ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto);
            // Validate that, Cluster exists with the name
            Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\".");
            emrCluster.setId(clusterSummary.getId());
            setEmrClusterStatus(emrCluster, clusterSummary.getStatus());
        }
        // Get active step details
        if (emrHelper.isActiveEmrState(emrCluster.getStatus())) {
            StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto);
            if (stepSummary != null) {
                EmrStep activeStep;
                // If verbose get active step details
                if (verbose) {
                    activeStep = buildEmrStepFromAwsStep(stepSummary, true);
                } else {
                    activeStep = buildEmrStepFromAwsStepSummary(stepSummary);
                }
                emrCluster.setActiveStep(activeStep);
            }
        }
        // Get requested step details
        if (StringUtils.isNotBlank(emrStepId)) {
            Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto);
            emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose));
        }
        // Get instance fleet if true
        if (BooleanUtils.isTrue(retrieveInstanceFleets)) {
            ListInstanceFleetsResult listInstanceFleetsResult = emrDao.getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto);
            emrCluster.setInstanceFleets(emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult));
        }
    } catch (AmazonServiceException ex) {
        handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\".");
    }
    return emrCluster;
}

Example 39 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class CreateEmrCluster method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    // Create the request.
    EmrClusterCreateRequest request = new EmrClusterCreateRequest();
    request.setNamespace(activitiHelper.getExpressionVariableAsString(namespace, execution));
    request.setEmrClusterDefinitionName(activitiHelper.getExpressionVariableAsString(emrClusterDefinitionName, execution));
    request.setEmrClusterName(activitiHelper.getExpressionVariableAsString(emrClusterName, execution));
    request.setDryRun(activitiHelper.getExpressionVariableAsBoolean(dryRun, execution, "dryRun", false, false));
    String contentTypeString = activitiHelper.getExpressionVariableAsString(contentType, execution);
    String emrClusterDefinitionOverrideString = activitiHelper.getExpressionVariableAsString(emrClusterDefinitionOverride, execution);
    if (StringUtils.isNotBlank(contentTypeString) && StringUtils.isBlank(emrClusterDefinitionOverrideString)) {
        throw new IllegalArgumentException("emrClusterDefinitionOverride is required when contentType is specified");
    } else if (StringUtils.isBlank(contentTypeString) && StringUtils.isNotBlank(emrClusterDefinitionOverrideString)) {
        throw new IllegalArgumentException("contentType is required when emrClusterDefinitionOverride is specified");
    } else if (StringUtils.isNotBlank(contentTypeString)) {
        EmrClusterDefinition emrClusterDefinitionOverride = getRequestObject(contentTypeString, emrClusterDefinitionOverrideString, EmrClusterDefinition.class);
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    }
    // Create the cluster in a new transaction.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Set workflow variables based on the result EMR cluster that was created.
    setResultWorkflowVariables(execution, emrCluster);
    LOGGER.info("{} EMR cluster started. emrClusterId=\"{}\"", activitiHelper.getProcessIdentifyingInformation(execution), emrCluster.getId());
}