Examples with EmrCluster org.finra.herd.model.api.xml.EmrCluster used on opensource projects

Example 1 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class TerminateEmrCluster method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = getClusterAlternateKey(execution);
    boolean overrideTerminationProtectionBoolean = activitiHelper.getExpressionVariableAsBoolean(overrideTerminationProtection, execution, "overrideTerminationProtection", false, false);
    String emrClusterIdString = activitiHelper.getExpressionVariableAsString(emrClusterId, execution);
    String accountIdString = activitiHelper.getExpressionVariableAsString(accountId, execution);
    // Terminate the EMR cluster.
    EmrCluster emrCluster = emrService.terminateCluster(emrClusterAlternateKeyDto, overrideTerminationProtectionBoolean, emrClusterIdString, accountIdString);
    // Set workflow variables based on the result EMR cluster that was terminated.
    setIdStatusWorkflowVariables(execution, emrCluster);
    LOGGER.info("{} EMR cluster terminated. emrClusterId=\"{}\"", activitiHelper.getProcessIdentifyingInformation(execution), emrCluster.getId());
}

Example 2 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class CheckEmrCluster method executeImpl.

@Override
public void executeImpl(DelegateExecution execution) throws Exception {
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = getClusterAlternateKey(execution);
    String emrStepIdString = activitiHelper.getExpressionVariableAsString(emrStepId, execution);
    String emrClusterIdString = activitiHelper.getExpressionVariableAsString(emrClusterId, execution);
    boolean verboseBoolean = activitiHelper.getExpressionVariableAsBoolean(verbose, execution, "verbose", false, false);
    String accountIdString = activitiHelper.getExpressionVariableAsString(accountId, execution);
    Boolean retrieveInstanceFleetsBoolean = activitiHelper.getExpressionVariableAsBoolean(retrieveInstanceFleets, execution, "retrieveInstanceFleets", false, false);
    // Gets the EMR cluster details.
    EmrCluster emrCluster = emrService.getCluster(emrClusterAlternateKeyDto, emrClusterIdString, emrStepIdString, verboseBoolean, accountIdString, retrieveInstanceFleetsBoolean);
    // Set cluster id and status workflow variables based on the result EMR cluster.
    setIdStatusWorkflowVariables(execution, emrCluster);
    setTaskWorkflowVariable(execution, VARIABLE_EMR_CLUSTER_STATUS_CHANGE_REASON_CODE, emrCluster.getStatusChangeReason().getCode());
    setTaskWorkflowVariable(execution, VARIABLE_EMR_CLUSTER_STATUS_CHANGE_REASON_MESSAGE, emrCluster.getStatusChangeReason().getMessage());
    setTaskWorkflowVariable(execution, VARIABLE_EMR_CLUSTER_CREATION_TIME, emrCluster.getStatusTimeline().getCreationTime());
    setTaskWorkflowVariable(execution, VARIABLE_EMR_CLUSTER_READY_TIME, emrCluster.getStatusTimeline().getReadyTime());
    setTaskWorkflowVariable(execution, VARIABLE_EMR_CLUSTER_END_TIME, emrCluster.getStatusTimeline().getEndTime());
    // Set the active step details in workflow variables
    if (emrCluster.getActiveStep() != null && emrCluster.getActiveStep().getId() != null) {
        setTaskWorkflowVariable(execution, "activeStep_id", emrCluster.getActiveStep().getId());
        setTaskWorkflowVariable(execution, "activeStep_stepName", emrCluster.getActiveStep().getStepName());
        setTaskWorkflowVariable(execution, "activeStep_status", emrCluster.getActiveStep().getStatus());
        if (verboseBoolean) {
            setTaskWorkflowVariable(execution, "activeStep_jarLocation", emrCluster.getActiveStep().getJarLocation());
            setTaskWorkflowVariable(execution, "activeStep_mainClass", emrCluster.getActiveStep().getMainClass());
            setTaskWorkflowVariable(execution, "activeStep_scriptArguments", herdStringHelper.buildStringWithDefaultDelimiter(emrCluster.getActiveStep().getScriptArguments()));
            setTaskWorkflowVariable(execution, "activeStep_continueOnError", emrCluster.getActiveStep().getContinueOnError());
        }
    }
    // Set the requested step details in workflow variables
    if (emrCluster.getStep() != null && emrCluster.getStep().getId() != null) {
        setTaskWorkflowVariable(execution, "step_id", emrCluster.getStep().getId());
        setTaskWorkflowVariable(execution, "step_stepName", emrCluster.getStep().getStepName());
        setTaskWorkflowVariable(execution, "step_status", emrCluster.getStep().getStatus());
        if (verboseBoolean) {
            setTaskWorkflowVariable(execution, "step_jarLocation", emrCluster.getStep().getJarLocation());
            setTaskWorkflowVariable(execution, "step_mainClass", emrCluster.getStep().getMainClass());
            setTaskWorkflowVariable(execution, "step_scriptArguments", herdStringHelper.buildStringWithDefaultDelimiter(emrCluster.getStep().getScriptArguments()));
            setTaskWorkflowVariable(execution, "step_continueOnError", emrCluster.getStep().getContinueOnError());
        }
    }
    // Set the instance fleets variable
    if (BooleanUtils.isTrue(retrieveInstanceFleetsBoolean)) {
        setTaskWorkflowVariable(execution, "instance_fleets", jsonHelper.objectToJson(emrCluster.getInstanceFleets()));
    }
}

Example 3 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrRestControllerTest method testCreateEmrCluster.

@Test
public void testCreateEmrCluster() throws Exception {
    // Create an EMR cluster definition override.
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    // Create an EMR cluster create request.
    EmrClusterCreateRequest emrClusterCreateRequest = new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, DRY_RUN, emrClusterDefinitionOverride);
    // Create an EMR cluster.
    EmrCluster emrCluster = new EmrCluster();
    emrCluster.setId(EMR_CLUSTER_ID);
    // Mock the external calls.
    when(emrService.createCluster(emrClusterCreateRequest)).thenReturn(emrCluster);
    // Call the method under test.
    EmrCluster result = emrRestController.createEmrCluster(emrClusterCreateRequest);
    // Verify the external calls.
    verify(emrService).createCluster(emrClusterCreateRequest);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(emrCluster, result);
}

Example 4 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrRestControllerTest method testGetEmrCluster.

@Test
public void testGetEmrCluster() throws Exception {
    // Create an EMR cluster key.
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = new EmrClusterAlternateKeyDto(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME);
    // Create an EMR cluster.
    EmrCluster emrCluster = new EmrCluster();
    emrCluster.setId(EMR_CLUSTER_ID);
    // Mock the external calls.
    when(emrService.getCluster(emrClusterAlternateKeyDto, EMR_CLUSTER_ID, EMR_STEP_ID, EMR_CLUSTER_VERBOSE_FLAG, AWS_ACCOUNT_ID, RETRIEVE_INSTANCE_FLEETS)).thenReturn(emrCluster);
    // Call the method under test.
    EmrCluster result = emrRestController.getEmrCluster(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_CLUSTER_ID, EMR_STEP_ID, EMR_CLUSTER_VERBOSE_FLAG, AWS_ACCOUNT_ID, RETRIEVE_INSTANCE_FLEETS);
    // Verify the external calls.
    verify(emrService).getCluster(emrClusterAlternateKeyDto, EMR_CLUSTER_ID, EMR_STEP_ID, EMR_CLUSTER_VERBOSE_FLAG, AWS_ACCOUNT_ID, RETRIEVE_INSTANCE_FLEETS);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(emrCluster, result);
}

Example 5 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideList.

@Test
public void testCreateEmrClusterOverrideList() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setApplications(Collections.<EmrClusterDefinitionApplication>emptyList());
    expectedEmrClusterDefinition.setApplications(emrClusterDefinitionOverride.getApplications());
    emrClusterDefinitionOverride.setConfigurations(Collections.<EmrClusterDefinitionConfiguration>emptyList());
    expectedEmrClusterDefinition.setConfigurations(emrClusterDefinitionOverride.getConfigurations());
    emrClusterDefinitionOverride.setCustomBootstrapActionAll(Collections.<ScriptDefinition>emptyList());
    expectedEmrClusterDefinition.setCustomBootstrapActionAll(emrClusterDefinitionOverride.getCustomBootstrapActionAll());
    emrClusterDefinitionOverride.setCustomBootstrapActionMaster(Collections.<ScriptDefinition>emptyList());
    expectedEmrClusterDefinition.setCustomBootstrapActionMaster(emrClusterDefinitionOverride.getCustomBootstrapActionMaster());
    emrClusterDefinitionOverride.setDaemonConfigurations(Collections.<Parameter>emptyList());
    expectedEmrClusterDefinition.setDaemonConfigurations(emrClusterDefinitionOverride.getDaemonConfigurations());
    emrClusterDefinitionOverride.setHadoopConfigurations(Collections.<Serializable>emptyList());
    expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
    emrClusterDefinitionOverride.setHadoopJarSteps(Collections.<HadoopJarStep>emptyList());
    expectedEmrClusterDefinition.setHadoopJarSteps(emrClusterDefinitionOverride.getHadoopJarSteps());
    emrClusterDefinitionOverride.setAdditionalMasterSecurityGroups(Collections.emptyList());
    expectedEmrClusterDefinition.setAdditionalMasterSecurityGroups(emrClusterDefinitionOverride.getAdditionalMasterSecurityGroups());
    emrClusterDefinitionOverride.setAdditionalSlaveSecurityGroups(Collections.emptyList());
    expectedEmrClusterDefinition.setAdditionalSlaveSecurityGroups(emrClusterDefinitionOverride.getAdditionalSlaveSecurityGroups());
    List<NodeTag> nodeTags = expectedEmrClusterDefinition.getNodeTags();
    nodeTags.add(new NodeTag("testTag", "test"));
    emrClusterDefinitionOverride.setNodeTags(nodeTags);
    expectedEmrClusterDefinition.setNodeTags(nodeTags);
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}