Examples with EmrClusterCreateRequest org.finra.herd.model.api.xml.EmrClusterCreateRequest used on opensource projects

Example 66 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceWithAccountIdTest method testCreateEmrClusterWithAccountId.

@Test
public void testCreateEmrClusterWithAccountId() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create a trusting AWS account.
    trustingAccountDaoTestHelper.createTrustingAccountEntity(AWS_ACCOUNT_ID, AWS_ROLE_ARN);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request and add an account id to the EMR cluster definition override.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequestWithAccountId();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    emrClusterDefinitionOverride.setAccountId(AWS_ACCOUNT_ID);
    // Create an EMR cluster.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Update the expected EMR cluster definition with the account id.
    expectedEmrClusterDefinition.setAccountId(AWS_ACCOUNT_ID);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertEquals(AWS_ACCOUNT_ID, emrCluster.getAccountId());
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 67 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrHelperTest method testEmrAddStepsAllTypesNegativeTestCase.

/**
 * This method tests the negative test cases scenario by testing all the step types
 */
@Test
public void testEmrAddStepsAllTypesNegativeTestCase() throws Exception {
    // Create a namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create an EMR cluster definition entity.
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create EC2 on-demand pricing entities.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    EmrStepHelper stepHelper;
    // Shell script arguments
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    List<Object> steps = new ArrayList<>();
    // Shell step parameters
    EmrShellStep shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStep.setStepName("Test Shell Script");
    shellStep.setScriptArguments(shellScriptArgs);
    shellStep.setContinueOnError(true);
    steps.add(shellStep);
    // Hive step parameters
    EmrHiveStep hiveStep = new EmrHiveStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    hiveStep.setStepName("Test Hive");
    hiveStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStep.setContinueOnError(true);
    steps.add(hiveStep);
    // Pig step parameter
    EmrPigStep pigStep = new EmrPigStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    pigStep.setStepName("Test Pig");
    pigStep.setContinueOnError(true);
    pigStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    steps.add(pigStep);
    // Oozie step that includes a shell script to install oozie
    shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStep.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStep.setScriptArguments(shellScriptArgsOozie);
    steps.add(shellStep);
    // Hadoop jar step configuration
    EmrHadoopJarStep hadoopJarStep = new EmrHadoopJarStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null, null);
    hadoopJarStep.setContinueOnError(true);
    hadoopJarStep.setStepName("Hadoop Jar");
    hadoopJarStep.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStep.setMainClass("wordcount");
    steps.add(hadoopJarStep);
    for (Object emrStep : steps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStep.getClass().getName());
        emrDao.addEmrStep(emrCluster.getId(), stepHelper.getEmrStepConfig(emrStep), emrHelper.getAwsParamsDto());
    }
}

Example 68 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class CreateEmrCluster method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    // Create the request.
    EmrClusterCreateRequest request = new EmrClusterCreateRequest();
    request.setNamespace(activitiHelper.getExpressionVariableAsString(namespace, execution));
    request.setEmrClusterDefinitionName(activitiHelper.getExpressionVariableAsString(emrClusterDefinitionName, execution));
    request.setEmrClusterName(activitiHelper.getExpressionVariableAsString(emrClusterName, execution));
    request.setDryRun(activitiHelper.getExpressionVariableAsBoolean(dryRun, execution, "dryRun", false, false));
    String contentTypeString = activitiHelper.getExpressionVariableAsString(contentType, execution);
    String emrClusterDefinitionOverrideString = activitiHelper.getExpressionVariableAsString(emrClusterDefinitionOverride, execution);
    if (StringUtils.isNotBlank(contentTypeString) && StringUtils.isBlank(emrClusterDefinitionOverrideString)) {
        throw new IllegalArgumentException("emrClusterDefinitionOverride is required when contentType is specified");
    } else if (StringUtils.isBlank(contentTypeString) && StringUtils.isNotBlank(emrClusterDefinitionOverrideString)) {
        throw new IllegalArgumentException("contentType is required when emrClusterDefinitionOverride is specified");
    } else if (StringUtils.isNotBlank(contentTypeString)) {
        EmrClusterDefinition emrClusterDefinitionOverride = getRequestObject(contentTypeString, emrClusterDefinitionOverrideString, EmrClusterDefinition.class);
        request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    }
    // Create the cluster in a new transaction.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Set workflow variables based on the result EMR cluster that was created.
    setResultWorkflowVariables(execution, emrCluster);
    LOGGER.info("{} EMR cluster started. emrClusterId=\"{}\"", activitiHelper.getProcessIdentifyingInformation(execution), emrCluster.getId());
}