Examples with EmrClusterCreateRequest org.finra.herd.model.api.xml.EmrClusterCreateRequest used on opensource projects

Example 46 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddStepsAllTypes.

/**
 * This method tests the happy path scenario by testing all the step types
 */
@Test
public void testEmrAddStepsAllTypes() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    List<Object> emrSteps = new ArrayList<>();
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    EmrShellStepAddRequest shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStepRequest.setStepName("Test Shell Script");
    shellStepRequest.setScriptArguments(shellScriptArgs);
    emrSteps.add(shellStepRequest);
    EmrHiveStepAddRequest hiveStepRequest = new EmrHiveStepAddRequest();
    List<String> scriptArgs1 = new ArrayList<>();
    scriptArgs1.add("arg2=sampleArg");
    scriptArgs1.add("arg1=tables");
    hiveStepRequest.setStepName("Test Hive");
    hiveStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStepRequest.setScriptArguments(scriptArgs1);
    emrSteps.add(hiveStepRequest);
    EmrPigStepAddRequest pigStepRequest = new EmrPigStepAddRequest();
    pigStepRequest.setStepName("Test Pig");
    pigStepRequest.setScriptArguments(shellScriptArgs);
    pigStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    emrSteps.add(pigStepRequest);
    shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStepRequest.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStepRequest.setScriptArguments(shellScriptArgsOozie);
    emrSteps.add(shellStepRequest);
    EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
    List<String> scriptArgs2 = new ArrayList<>();
    scriptArgs2.add("oozie_run");
    scriptArgs2.add("wordcountOutput");
    hadoopJarStepRequest.setStepName("Hadoop Jar");
    hadoopJarStepRequest.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStepRequest.setMainClass("wordcount");
    hadoopJarStepRequest.setScriptArguments(scriptArgs2);
    emrSteps.add(hadoopJarStepRequest);
    EmrStepHelper stepHelper;
    for (Object emrStepAddRequest : emrSteps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStepAddRequest.getClass().getName());
        stepHelper.setRequestNamespace(emrStepAddRequest, NAMESPACE);
        stepHelper.setRequestEmrClusterDefinitionName(emrStepAddRequest, EMR_CLUSTER_DEFINITION_NAME);
        stepHelper.setRequestEmrClusterName(emrStepAddRequest, request.getEmrClusterName());
        Object emrStep = emrService.addStepToCluster(emrStepAddRequest);
        assertNotNull(emrStep);
        assertNotNull(stepHelper.getStepId(emrStep));
        Method getNameMethod = emrStep.getClass().getMethod("getStepName");
        String emrStepName = (String) getNameMethod.invoke(emrStep);
        assertEquals(stepHelper.getRequestStepName(emrStepAddRequest), emrStepName);
        Method isContinueOnErrorMethod = emrStep.getClass().getMethod("isContinueOnError");
        Object emrStepIsContinueOnError = isContinueOnErrorMethod.invoke(emrStep);
        assertEquals(stepHelper.isRequestContinueOnError(emrStepAddRequest), emrStepIsContinueOnError);
    }
}

Example 47 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideScalar.

@Test
public void testCreateEmrClusterOverrideScalar() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    // Security configuration is not supported prior to EMR version 4.8.0.
    emrClusterDefinitionOverride.setReleaseLabel("4.8.0." + Math.random());
    expectedEmrClusterDefinition.setReleaseLabel(emrClusterDefinitionOverride.getReleaseLabel());
    emrClusterDefinitionOverride.setAdditionalInfo("test" + Math.random());
    expectedEmrClusterDefinition.setAdditionalInfo(emrClusterDefinitionOverride.getAdditionalInfo());
    emrClusterDefinitionOverride.setAmiVersion("test" + Math.random());
    expectedEmrClusterDefinition.setAmiVersion(emrClusterDefinitionOverride.getAmiVersion());
    emrClusterDefinitionOverride.setEc2NodeIamProfileName("test" + Math.random());
    expectedEmrClusterDefinition.setEc2NodeIamProfileName(emrClusterDefinitionOverride.getEc2NodeIamProfileName());
    emrClusterDefinitionOverride.setEncryptionEnabled(!expectedEmrClusterDefinition.isEncryptionEnabled());
    expectedEmrClusterDefinition.setEncryptionEnabled(emrClusterDefinitionOverride.isEncryptionEnabled());
    emrClusterDefinitionOverride.setHadoopVersion("test" + Math.random());
    expectedEmrClusterDefinition.setHadoopVersion(emrClusterDefinitionOverride.getHadoopVersion());
    emrClusterDefinitionOverride.setHiveVersion("test" + Math.random());
    expectedEmrClusterDefinition.setHiveVersion(emrClusterDefinitionOverride.getHiveVersion());
    emrClusterDefinitionOverride.setInstallOozie(!expectedEmrClusterDefinition.isInstallOozie());
    expectedEmrClusterDefinition.setInstallOozie(emrClusterDefinitionOverride.isInstallOozie());
    emrClusterDefinitionOverride.setKeepAlive(!expectedEmrClusterDefinition.isKeepAlive());
    expectedEmrClusterDefinition.setKeepAlive(emrClusterDefinitionOverride.isKeepAlive());
    emrClusterDefinitionOverride.setLogBucket("test" + Math.random());
    expectedEmrClusterDefinition.setLogBucket(emrClusterDefinitionOverride.getLogBucket());
    emrClusterDefinitionOverride.setPigVersion("test" + Math.random());
    expectedEmrClusterDefinition.setPigVersion(emrClusterDefinitionOverride.getPigVersion());
    emrClusterDefinitionOverride.setServiceIamRole("test" + Math.random());
    expectedEmrClusterDefinition.setServiceIamRole(emrClusterDefinitionOverride.getServiceIamRole());
    emrClusterDefinitionOverride.setSshKeyPairName("test" + Math.random());
    expectedEmrClusterDefinition.setSshKeyPairName(emrClusterDefinitionOverride.getSshKeyPairName());
    emrClusterDefinitionOverride.setSubnetId(MockEc2OperationsImpl.SUBNET_1);
    expectedEmrClusterDefinition.setSubnetId(emrClusterDefinitionOverride.getSubnetId());
    emrClusterDefinitionOverride.setSupportedProduct("test" + Math.random());
    expectedEmrClusterDefinition.setSupportedProduct(emrClusterDefinitionOverride.getSupportedProduct());
    emrClusterDefinitionOverride.setSecurityConfiguration("test" + Math.random());
    expectedEmrClusterDefinition.setSecurityConfiguration(emrClusterDefinitionOverride.getSecurityConfiguration());
    emrClusterDefinitionOverride.setScaleDownBehavior("test" + Math.random());
    expectedEmrClusterDefinition.setScaleDownBehavior(emrClusterDefinitionOverride.getScaleDownBehavior());
    emrClusterDefinitionOverride.setTerminationProtection(!expectedEmrClusterDefinition.isTerminationProtection());
    expectedEmrClusterDefinition.setTerminationProtection(emrClusterDefinitionOverride.isTerminationProtection());
    emrClusterDefinitionOverride.setVisibleToAll(!expectedEmrClusterDefinition.isVisibleToAll());
    expectedEmrClusterDefinition.setVisibleToAll(emrClusterDefinitionOverride.isVisibleToAll());
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 48 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddSteps.

/**
 * This method tests the happy path scenario by providing all the parameters
 */
@Test
public void testEmrAddSteps() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    // Create the Add step request.
    EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
    EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);
    // Validate the returned object against the input.
    assertNotNull(emrShellStep);
    assertTrue(emrShellStep.getNamespace().equals(request.getNamespace()));
    assertTrue(emrShellStep.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrShellStep.getEmrClusterName().equals(request.getEmrClusterName()));
}

Example 49 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testAddSecurityGroupEC2InstanceNotProvisioned.

/**
 * This method tests the scenario where EC2 instances have not been provisioned yet
 */
@Test(expected = IllegalArgumentException.class)
public void testAddSecurityGroupEC2InstanceNotProvisioned() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    request.setEmrClusterName(MockEmrOperationsImpl.MOCK_CLUSTER_NOT_PROVISIONED_NAME);
    emrService.createCluster(request);
    // Create the Add security group.
    EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(request.getEmrClusterName());
    emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
}

Example 50 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideExistingCoreInstanceToNullAssertSuccess.

@Test
public void testCreateEmrClusterOverrideExistingCoreInstanceToNullAssertSuccess() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinition.getInstanceDefinitions().getCoreInstances().setInstanceCount(1);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, xmlHelper.objectToXml(emrClusterDefinition));
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setInstanceDefinitions(emrClusterDefinition.getInstanceDefinitions());
    emrClusterDefinitionOverride.getInstanceDefinitions().setCoreInstances(null);
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    EmrCluster emrClusterCreateResponse = emrService.createCluster(request);
    assertNull(emrClusterCreateResponse.getEmrClusterDefinition().getInstanceDefinitions().getCoreInstances());
}