Examples with EmrClusterDefinitionEntity org.finra.herd.model.jpa.EmrClusterDefinitionEntity used on opensource projects

Example 26 with EmrClusterDefinitionEntity

use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.

the class EmrClusterDefinitionServiceImpl method createEmrClusterDefinition.

@NamespacePermission(fields = "#request?.emrClusterDefinitionKey?.namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public EmrClusterDefinitionInformation createEmrClusterDefinition(EmrClusterDefinitionCreateRequest request) throws Exception {
    // Perform validate and trim of the EMR cluster definition key.
    emrClusterDefinitionHelper.validateEmrClusterDefinitionKey(request.getEmrClusterDefinitionKey());
    // Validate the EMR cluster definition configuration.
    emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(request.getEmrClusterDefinition());
    // Get the namespace and ensure it exists.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(request.getEmrClusterDefinitionKey().getNamespace());
    namespaceIamRoleAuthorizationHelper.checkPermissions(namespaceEntity, request.getEmrClusterDefinition().getServiceIamRole(), request.getEmrClusterDefinition().getEc2NodeIamProfileName());
    // Ensure a EMR cluster definition with the specified name doesn't already exist.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDao.getEmrClusterDefinitionByAltKey(request.getEmrClusterDefinitionKey());
    if (emrClusterDefinitionEntity != null) {
        throw new AlreadyExistsException(String.format("Unable to create EMR cluster definition with name \"%s\" for namespace \"%s\" because it already exists.", request.getEmrClusterDefinitionKey().getEmrClusterDefinitionName(), request.getEmrClusterDefinitionKey().getNamespace()));
    }
    // Create a EMR cluster definition entity from the request information.
    emrClusterDefinitionEntity = createEmrClusterDefinitionEntity(namespaceEntity, request);
    // Persist the new entity.
    emrClusterDefinitionEntity = emrClusterDefinitionDao.saveAndRefresh(emrClusterDefinitionEntity);
    // Create and return the EMR cluster definition object from the persisted entity.
    return createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity);
}

Example 27 with EmrClusterDefinitionEntity

use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.

the class EmrClusterDefinitionServiceImpl method updateEmrClusterDefinition.

@NamespacePermission(fields = "#emrClusterDefinitionKey?.namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public EmrClusterDefinitionInformation updateEmrClusterDefinition(EmrClusterDefinitionKey emrClusterDefinitionKey, EmrClusterDefinitionUpdateRequest request) throws Exception {
    // Perform validate and trim of the EMR cluster definition key.
    emrClusterDefinitionHelper.validateEmrClusterDefinitionKey(emrClusterDefinitionKey);
    // Validate the EMR cluster definition configuration.
    emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(request.getEmrClusterDefinition());
    // Retrieve and ensure that a EMR cluster definition already exists with the specified name.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterDefinitionKey);
    namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), request.getEmrClusterDefinition().getServiceIamRole(), request.getEmrClusterDefinition().getEc2NodeIamProfileName());
    // Log the existing EMR cluster definition before the update.
    LOGGER.info("Logging EMR cluster definition before the update. emrClusterDefinition={}", xmlHelper.objectToXml(createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity), true));
    // Convert EMR cluster configuration to the XML representation.
    String emrClusterConfiguration = xmlHelper.objectToXml(request.getEmrClusterDefinition());
    // Update the EMR cluster definition entity.
    emrClusterDefinitionEntity.setConfiguration(emrClusterConfiguration);
    // Persist and refresh the entity.
    emrClusterDefinitionEntity = emrClusterDefinitionDao.saveAndRefresh(emrClusterDefinitionEntity);
    // Create and return the EMR cluster definition object from the persisted entity.
    return createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity);
}

Example 28 with EmrClusterDefinitionEntity

use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.

the class EmrServiceImpl method addSecurityGroupsToClusterMasterImpl.

/**
 * Adds security groups to the master node of an existing EMR Cluster.
 *
 * @param request the EMR master security group add request
 *
 * @return the added EMR master security groups
 * @throws Exception if there were any errors adding the security groups to the cluster master.
 */
protected EmrMasterSecurityGroup addSecurityGroupsToClusterMasterImpl(EmrMasterSecurityGroupAddRequest request) throws Exception {
    // Perform the request validation.
    validateAddSecurityGroupsToClusterMasterRequest(request);
    // Get account and AwsParamDto
    String accountId = request.getAccountId();
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(accountId);
    // Get the namespace and ensure it exists.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(request.getNamespace());
    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(request.getNamespace(), request.getEmrClusterDefinitionName());
    List<String> groupIds = null;
    String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), request.getEmrClusterName());
    try {
        groupIds = emrDao.addEmrMasterSecurityGroups(emrHelper.getActiveEmrClusterId(request.getEmrClusterId(), clusterName, request.getAccountId()), request.getSecurityGroupIds(), awsParamsDto);
    } catch (AmazonServiceException ex) {
        handleAmazonException(ex, "An Amazon exception occurred while adding EMR security groups: " + herdStringHelper.buildStringWithDefaultDelimiter(request.getSecurityGroupIds()) + " to cluster: " + clusterName);
    }
    return createEmrClusterMasterGroupFromRequest(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), request.getEmrClusterName(), groupIds);
}

Example 29 with EmrClusterDefinitionEntity

use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.

the class EmrServiceImpl method getClusterImpl.

/**
 * Gets details of an existing EMR Cluster.
 *
 * @param emrClusterAlternateKeyDto the EMR cluster alternate key
 * @param emrClusterId the cluster id of the cluster to get details
 * @param emrStepId the step id of the step to get details
 * @param verbose parameter for whether to return detailed information
 * @param accountId the optional AWS account that EMR cluster is running in
 * @param retrieveInstanceFleets parameter for whether to retrieve instance fleets
 *
 * @return the EMR Cluster object with details.
 * @throws Exception if an error occurred while getting the cluster
 */
protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, String accountId, Boolean retrieveInstanceFleets) throws Exception {
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(accountId);
    // Perform the request validation.
    validateEmrClusterKey(emrClusterAlternateKeyDto);
    // Get the namespace and ensure it exists.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace());
    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName());
    EmrCluster emrCluster = createEmrClusterFromRequest(null, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null);
    String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName());
    try {
        // Get Cluster status if clusterId is specified
        if (StringUtils.isNotBlank(emrClusterId)) {
            Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto);
            // Validate that, Cluster exists
            Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\".");
            // Validate that, Cluster name match as specified
            Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified.");
            emrCluster.setId(cluster.getId());
            setEmrClusterStatus(emrCluster, cluster.getStatus());
        } else {
            ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto);
            // Validate that, Cluster exists with the name
            Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\".");
            emrCluster.setId(clusterSummary.getId());
            setEmrClusterStatus(emrCluster, clusterSummary.getStatus());
        }
        // Get active step details
        if (emrHelper.isActiveEmrState(emrCluster.getStatus())) {
            StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto);
            if (stepSummary != null) {
                EmrStep activeStep;
                // If verbose get active step details
                if (verbose) {
                    activeStep = buildEmrStepFromAwsStep(stepSummary, true);
                } else {
                    activeStep = buildEmrStepFromAwsStepSummary(stepSummary);
                }
                emrCluster.setActiveStep(activeStep);
            }
        }
        // Get requested step details
        if (StringUtils.isNotBlank(emrStepId)) {
            Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto);
            emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose));
        }
        // Get instance fleet if true
        if (BooleanUtils.isTrue(retrieveInstanceFleets)) {
            ListInstanceFleetsResult listInstanceFleetsResult = emrDao.getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto);
            emrCluster.setInstanceFleets(emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult));
        }
    } catch (AmazonServiceException ex) {
        handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\".");
    }
    return emrCluster;
}

Example 30 with EmrClusterDefinitionEntity

use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.

the class EmrServiceImpl method createClusterImpl.

/**
 * <p> Creates a new EMR cluster based on the given request if the cluster with the given name does not already exist. If the cluster already exist, returns
 * the information about the existing cluster. </p> <p> The request must contain: </p> <ul> <li>A namespace and definition name which refer to an existing
 * EMR cluster definition.</li> <li>A valid cluster name to create.</li> </ul> <p> The request may optionally contain: </p> <ul> <li>A "dry run" flag, which
 * when set to {@code true}, no calls to AWS will occur, but validations and override will. Defaults to {@code false}.</li> <li>An override parameter, which
 * when set, overrides the given parameters in the cluster definition before creating the cluster. Defaults to no override. </li> </ul> <p> A successful
 * response will contain: </p> <ul> <li>The ID of the cluster that was created, or if the cluster already exists, the ID of the cluster that exists. This
 * field will be {@code null} when dry run flag is {@code true}.</li> <li>The status of the cluster that was created or already exists. The status will
 * normally be "Starting" on successful creations. This field will be {@code null} when dry run flag is {@code true}</li> <li>The namespace, definition
 * name, and cluster name of the cluster.</li> <li>The dry run flag, if given in the request.</li> <li>An indicator whether the cluster was created or not.
 * If the cluster already exists, the cluster will not be created and this flag will be set to {@code false}.</li> <li>The definition which was used to
 * create the cluster. If any overrides were given, this definition's values will be the values after the override. This field will be {@code null} if the
 * cluster was not created.</li> </ul> <p> Notes: </p> <ul> <li>At any point of the execution, if there are validation errors, the method will immediately
 * throw an exception.</li> <li>Even if the validations pass, AWS may still reject the request, which will cause this method to throw an exception.</li>
 * <li>Dry runs do not make any calls to AWS, therefore AWS may still reject the creation request even when a dry run succeeds.</li> </ul>
 *
 * @param request - {@link EmrClusterCreateRequest} The EMR cluster create request
 *
 * @return {@link EmrCluster} the created EMR cluster object
 * @throws Exception when the original EMR cluster definition XML is malformed
 */
protected EmrCluster createClusterImpl(EmrClusterCreateRequest request) throws Exception {
    // Extract EMR cluster alternate key from the create request.
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = getEmrClusterAlternateKey(request);
    // Perform the request validation.
    validateEmrClusterKey(emrClusterAlternateKeyDto);
    // Get the namespace and ensure it exists.
    NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace());
    // Get the EMR cluster definition and ensure it exists.
    EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName());
    // Replace all S3 managed location variables in xml
    String toReplace = getS3ManagedReplaceString();
    String replacedConfigXml = emrClusterDefinitionEntity.getConfiguration().replaceAll(toReplace, emrHelper.getS3StagingLocation());
    // Unmarshal definition xml into JAXB object.
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, replacedConfigXml);
    // Perform override if override is set.
    overrideEmrClusterDefinition(emrClusterDefinition, request.getEmrClusterDefinitionOverride());
    // Perform the EMR cluster definition configuration validation.
    emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(emrClusterDefinition);
    // Check permissions.
    namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), emrClusterDefinition.getServiceIamRole(), emrClusterDefinition.getEc2NodeIamProfileName());
    AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(emrClusterDefinition.getAccountId());
    // If instance group definitions are specified, find best price and update definition.
    if (!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())) {
        emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
    }
    // The cluster ID record.
    String clusterId = null;
    String emrClusterStatus = null;
    String accountId = emrClusterDefinition.getAccountId();
    // Was cluster created?
    Boolean emrClusterCreated = null;
    // If the dryRun flag is null or false. This is the default option if no flag is given.
    if (!Boolean.TRUE.equals(request.isDryRun())) {
        /*
             * Create the cluster only if the cluster does not already exist.
             * If the cluster is created, record the newly created cluster ID.
             * If the cluster already exists, record the existing cluster ID.
             * If there is any error while attempting to check for existing cluster or create a new one, handle the exception to throw appropriate exception.
             */
        String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName());
        try {
            // Try to get an active EMR cluster by its name.
            ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto);
            // If cluster does not already exist.
            if (clusterSummary == null) {
                clusterId = emrDao.createEmrCluster(clusterName, emrClusterDefinition, awsParamsDto);
                emrClusterCreated = true;
                EmrClusterCreationLogEntity emrClusterCreationLogEntity = new EmrClusterCreationLogEntity();
                emrClusterCreationLogEntity.setNamespace(emrClusterDefinitionEntity.getNamespace());
                emrClusterCreationLogEntity.setEmrClusterDefinitionName(emrClusterDefinitionEntity.getName());
                emrClusterCreationLogEntity.setEmrClusterName(emrClusterAlternateKeyDto.getEmrClusterName());
                emrClusterCreationLogEntity.setEmrClusterId(clusterId);
                emrClusterCreationLogEntity.setEmrClusterDefinition(xmlHelper.objectToXml(emrClusterDefinition));
                herdDao.saveAndRefresh(emrClusterCreationLogEntity);
            } else // If the cluster already exist.
            {
                clusterId = clusterSummary.getId();
                emrClusterCreated = false;
                // Do not include definition in response
                emrClusterDefinition = null;
            }
            emrClusterStatus = emrDao.getEmrClusterStatusById(clusterId, awsParamsDto);
        } catch (AmazonServiceException ex) {
            handleAmazonException(ex, "An Amazon exception occurred while creating EMR cluster with name \"" + clusterName + "\".");
        }
    } else // If the dryRun flag is true and not null
    {
        emrClusterCreated = false;
    }
    return createEmrClusterFromRequest(clusterId, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, emrClusterStatus, emrClusterCreated, request.isDryRun(), emrClusterDefinition);
}