use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.
the class EmrClusterDefinitionServiceImpl method createEmrClusterDefinition.
@NamespacePermission(fields = "#request?.emrClusterDefinitionKey?.namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public EmrClusterDefinitionInformation createEmrClusterDefinition(EmrClusterDefinitionCreateRequest request) throws Exception {
// Perform validate and trim of the EMR cluster definition key.
emrClusterDefinitionHelper.validateEmrClusterDefinitionKey(request.getEmrClusterDefinitionKey());
// Validate the EMR cluster definition configuration.
emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(request.getEmrClusterDefinition());
// Get the namespace and ensure it exists.
NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(request.getEmrClusterDefinitionKey().getNamespace());
namespaceIamRoleAuthorizationHelper.checkPermissions(namespaceEntity, request.getEmrClusterDefinition().getServiceIamRole(), request.getEmrClusterDefinition().getEc2NodeIamProfileName());
// Ensure a EMR cluster definition with the specified name doesn't already exist.
EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDao.getEmrClusterDefinitionByAltKey(request.getEmrClusterDefinitionKey());
if (emrClusterDefinitionEntity != null) {
throw new AlreadyExistsException(String.format("Unable to create EMR cluster definition with name \"%s\" for namespace \"%s\" because it already exists.", request.getEmrClusterDefinitionKey().getEmrClusterDefinitionName(), request.getEmrClusterDefinitionKey().getNamespace()));
}
// Create a EMR cluster definition entity from the request information.
emrClusterDefinitionEntity = createEmrClusterDefinitionEntity(namespaceEntity, request);
// Persist the new entity.
emrClusterDefinitionEntity = emrClusterDefinitionDao.saveAndRefresh(emrClusterDefinitionEntity);
// Create and return the EMR cluster definition object from the persisted entity.
return createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity);
}
use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.
the class EmrClusterDefinitionServiceImpl method updateEmrClusterDefinition.
@NamespacePermission(fields = "#emrClusterDefinitionKey?.namespace", permissions = NamespacePermissionEnum.WRITE)
@Override
public EmrClusterDefinitionInformation updateEmrClusterDefinition(EmrClusterDefinitionKey emrClusterDefinitionKey, EmrClusterDefinitionUpdateRequest request) throws Exception {
// Perform validate and trim of the EMR cluster definition key.
emrClusterDefinitionHelper.validateEmrClusterDefinitionKey(emrClusterDefinitionKey);
// Validate the EMR cluster definition configuration.
emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(request.getEmrClusterDefinition());
// Retrieve and ensure that a EMR cluster definition already exists with the specified name.
EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterDefinitionKey);
namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), request.getEmrClusterDefinition().getServiceIamRole(), request.getEmrClusterDefinition().getEc2NodeIamProfileName());
// Log the existing EMR cluster definition before the update.
LOGGER.info("Logging EMR cluster definition before the update. emrClusterDefinition={}", xmlHelper.objectToXml(createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity), true));
// Convert EMR cluster configuration to the XML representation.
String emrClusterConfiguration = xmlHelper.objectToXml(request.getEmrClusterDefinition());
// Update the EMR cluster definition entity.
emrClusterDefinitionEntity.setConfiguration(emrClusterConfiguration);
// Persist and refresh the entity.
emrClusterDefinitionEntity = emrClusterDefinitionDao.saveAndRefresh(emrClusterDefinitionEntity);
// Create and return the EMR cluster definition object from the persisted entity.
return createEmrClusterDefinitionFromEntity(emrClusterDefinitionEntity);
}
use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.
the class EmrServiceImpl method addSecurityGroupsToClusterMasterImpl.
/**
* Adds security groups to the master node of an existing EMR Cluster.
*
* @param request the EMR master security group add request
*
* @return the added EMR master security groups
* @throws Exception if there were any errors adding the security groups to the cluster master.
*/
protected EmrMasterSecurityGroup addSecurityGroupsToClusterMasterImpl(EmrMasterSecurityGroupAddRequest request) throws Exception {
// Perform the request validation.
validateAddSecurityGroupsToClusterMasterRequest(request);
// Get account and AwsParamDto
String accountId = request.getAccountId();
AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(accountId);
// Get the namespace and ensure it exists.
NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(request.getNamespace());
// Get the EMR cluster definition and ensure it exists.
EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(request.getNamespace(), request.getEmrClusterDefinitionName());
List<String> groupIds = null;
String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), request.getEmrClusterName());
try {
groupIds = emrDao.addEmrMasterSecurityGroups(emrHelper.getActiveEmrClusterId(request.getEmrClusterId(), clusterName, request.getAccountId()), request.getSecurityGroupIds(), awsParamsDto);
} catch (AmazonServiceException ex) {
handleAmazonException(ex, "An Amazon exception occurred while adding EMR security groups: " + herdStringHelper.buildStringWithDefaultDelimiter(request.getSecurityGroupIds()) + " to cluster: " + clusterName);
}
return createEmrClusterMasterGroupFromRequest(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), request.getEmrClusterName(), groupIds);
}
use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.
the class EmrServiceImpl method getClusterImpl.
/**
* Gets details of an existing EMR Cluster.
*
* @param emrClusterAlternateKeyDto the EMR cluster alternate key
* @param emrClusterId the cluster id of the cluster to get details
* @param emrStepId the step id of the step to get details
* @param verbose parameter for whether to return detailed information
* @param accountId the optional AWS account that EMR cluster is running in
* @param retrieveInstanceFleets parameter for whether to retrieve instance fleets
*
* @return the EMR Cluster object with details.
* @throws Exception if an error occurred while getting the cluster
*/
protected EmrCluster getClusterImpl(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, String emrClusterId, String emrStepId, boolean verbose, String accountId, Boolean retrieveInstanceFleets) throws Exception {
AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(accountId);
// Perform the request validation.
validateEmrClusterKey(emrClusterAlternateKeyDto);
// Get the namespace and ensure it exists.
NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace());
// Get the EMR cluster definition and ensure it exists.
EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName());
EmrCluster emrCluster = createEmrClusterFromRequest(null, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, null, null, null, null);
String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName());
try {
// Get Cluster status if clusterId is specified
if (StringUtils.isNotBlank(emrClusterId)) {
Cluster cluster = emrDao.getEmrClusterById(emrClusterId.trim(), awsParamsDto);
// Validate that, Cluster exists
Assert.notNull(cluster, "An EMR cluster must exists with the cluster ID \"" + emrClusterId + "\".");
// Validate that, Cluster name match as specified
Assert.isTrue(clusterName.equalsIgnoreCase(cluster.getName()), "Cluster name of specified cluster id \"" + emrClusterId + "\" must match the name specified.");
emrCluster.setId(cluster.getId());
setEmrClusterStatus(emrCluster, cluster.getStatus());
} else {
ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto);
// Validate that, Cluster exists with the name
Assert.notNull(clusterSummary, "An EMR cluster must exists with the name \"" + clusterName + "\".");
emrCluster.setId(clusterSummary.getId());
setEmrClusterStatus(emrCluster, clusterSummary.getStatus());
}
// Get active step details
if (emrHelper.isActiveEmrState(emrCluster.getStatus())) {
StepSummary stepSummary = emrDao.getClusterActiveStep(emrCluster.getId(), awsParamsDto);
if (stepSummary != null) {
EmrStep activeStep;
// If verbose get active step details
if (verbose) {
activeStep = buildEmrStepFromAwsStep(stepSummary, true);
} else {
activeStep = buildEmrStepFromAwsStepSummary(stepSummary);
}
emrCluster.setActiveStep(activeStep);
}
}
// Get requested step details
if (StringUtils.isNotBlank(emrStepId)) {
Step step = emrDao.getClusterStep(emrCluster.getId(), emrStepId.trim(), awsParamsDto);
emrCluster.setStep(buildEmrStepFromAwsStep(step, verbose));
}
// Get instance fleet if true
if (BooleanUtils.isTrue(retrieveInstanceFleets)) {
ListInstanceFleetsResult listInstanceFleetsResult = emrDao.getListInstanceFleetsResult(emrCluster.getId(), awsParamsDto);
emrCluster.setInstanceFleets(emrHelper.buildEmrClusterInstanceFleetFromAwsResult(listInstanceFleetsResult));
}
} catch (AmazonServiceException ex) {
handleAmazonException(ex, "An Amazon exception occurred while getting EMR cluster details with name \"" + clusterName + "\".");
}
return emrCluster;
}
use of org.finra.herd.model.jpa.EmrClusterDefinitionEntity in project herd by FINRAOS.
the class EmrServiceImpl method createClusterImpl.
/**
* <p> Creates a new EMR cluster based on the given request if the cluster with the given name does not already exist. If the cluster already exist, returns
* the information about the existing cluster. </p> <p> The request must contain: </p> <ul> <li>A namespace and definition name which refer to an existing
* EMR cluster definition.</li> <li>A valid cluster name to create.</li> </ul> <p> The request may optionally contain: </p> <ul> <li>A "dry run" flag, which
* when set to {@code true}, no calls to AWS will occur, but validations and override will. Defaults to {@code false}.</li> <li>An override parameter, which
* when set, overrides the given parameters in the cluster definition before creating the cluster. Defaults to no override. </li> </ul> <p> A successful
* response will contain: </p> <ul> <li>The ID of the cluster that was created, or if the cluster already exists, the ID of the cluster that exists. This
* field will be {@code null} when dry run flag is {@code true}.</li> <li>The status of the cluster that was created or already exists. The status will
* normally be "Starting" on successful creations. This field will be {@code null} when dry run flag is {@code true}</li> <li>The namespace, definition
* name, and cluster name of the cluster.</li> <li>The dry run flag, if given in the request.</li> <li>An indicator whether the cluster was created or not.
* If the cluster already exists, the cluster will not be created and this flag will be set to {@code false}.</li> <li>The definition which was used to
* create the cluster. If any overrides were given, this definition's values will be the values after the override. This field will be {@code null} if the
* cluster was not created.</li> </ul> <p> Notes: </p> <ul> <li>At any point of the execution, if there are validation errors, the method will immediately
* throw an exception.</li> <li>Even if the validations pass, AWS may still reject the request, which will cause this method to throw an exception.</li>
* <li>Dry runs do not make any calls to AWS, therefore AWS may still reject the creation request even when a dry run succeeds.</li> </ul>
*
* @param request - {@link EmrClusterCreateRequest} The EMR cluster create request
*
* @return {@link EmrCluster} the created EMR cluster object
* @throws Exception when the original EMR cluster definition XML is malformed
*/
protected EmrCluster createClusterImpl(EmrClusterCreateRequest request) throws Exception {
// Extract EMR cluster alternate key from the create request.
EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = getEmrClusterAlternateKey(request);
// Perform the request validation.
validateEmrClusterKey(emrClusterAlternateKeyDto);
// Get the namespace and ensure it exists.
NamespaceEntity namespaceEntity = namespaceDaoHelper.getNamespaceEntity(emrClusterAlternateKeyDto.getNamespace());
// Get the EMR cluster definition and ensure it exists.
EmrClusterDefinitionEntity emrClusterDefinitionEntity = emrClusterDefinitionDaoHelper.getEmrClusterDefinitionEntity(emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName());
// Replace all S3 managed location variables in xml
String toReplace = getS3ManagedReplaceString();
String replacedConfigXml = emrClusterDefinitionEntity.getConfiguration().replaceAll(toReplace, emrHelper.getS3StagingLocation());
// Unmarshal definition xml into JAXB object.
EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, replacedConfigXml);
// Perform override if override is set.
overrideEmrClusterDefinition(emrClusterDefinition, request.getEmrClusterDefinitionOverride());
// Perform the EMR cluster definition configuration validation.
emrClusterDefinitionHelper.validateEmrClusterDefinitionConfiguration(emrClusterDefinition);
// Check permissions.
namespaceIamRoleAuthorizationHelper.checkPermissions(emrClusterDefinitionEntity.getNamespace(), emrClusterDefinition.getServiceIamRole(), emrClusterDefinition.getEc2NodeIamProfileName());
AwsParamsDto awsParamsDto = emrHelper.getAwsParamsDtoByAcccountId(emrClusterDefinition.getAccountId());
// If instance group definitions are specified, find best price and update definition.
if (!emrHelper.isInstanceDefinitionsEmpty(emrClusterDefinition.getInstanceDefinitions())) {
emrPricingHelper.updateEmrClusterDefinitionWithBestPrice(emrClusterAlternateKeyDto, emrClusterDefinition, awsParamsDto);
}
// The cluster ID record.
String clusterId = null;
String emrClusterStatus = null;
String accountId = emrClusterDefinition.getAccountId();
// Was cluster created?
Boolean emrClusterCreated = null;
// If the dryRun flag is null or false. This is the default option if no flag is given.
if (!Boolean.TRUE.equals(request.isDryRun())) {
/*
* Create the cluster only if the cluster does not already exist.
* If the cluster is created, record the newly created cluster ID.
* If the cluster already exists, record the existing cluster ID.
* If there is any error while attempting to check for existing cluster or create a new one, handle the exception to throw appropriate exception.
*/
String clusterName = emrHelper.buildEmrClusterName(namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName());
try {
// Try to get an active EMR cluster by its name.
ClusterSummary clusterSummary = emrDao.getActiveEmrClusterByName(clusterName, awsParamsDto);
// If cluster does not already exist.
if (clusterSummary == null) {
clusterId = emrDao.createEmrCluster(clusterName, emrClusterDefinition, awsParamsDto);
emrClusterCreated = true;
EmrClusterCreationLogEntity emrClusterCreationLogEntity = new EmrClusterCreationLogEntity();
emrClusterCreationLogEntity.setNamespace(emrClusterDefinitionEntity.getNamespace());
emrClusterCreationLogEntity.setEmrClusterDefinitionName(emrClusterDefinitionEntity.getName());
emrClusterCreationLogEntity.setEmrClusterName(emrClusterAlternateKeyDto.getEmrClusterName());
emrClusterCreationLogEntity.setEmrClusterId(clusterId);
emrClusterCreationLogEntity.setEmrClusterDefinition(xmlHelper.objectToXml(emrClusterDefinition));
herdDao.saveAndRefresh(emrClusterCreationLogEntity);
} else // If the cluster already exist.
{
clusterId = clusterSummary.getId();
emrClusterCreated = false;
// Do not include definition in response
emrClusterDefinition = null;
}
emrClusterStatus = emrDao.getEmrClusterStatusById(clusterId, awsParamsDto);
} catch (AmazonServiceException ex) {
handleAmazonException(ex, "An Amazon exception occurred while creating EMR cluster with name \"" + clusterName + "\".");
}
} else // If the dryRun flag is true and not null
{
emrClusterCreated = false;
}
return createEmrClusterFromRequest(clusterId, namespaceEntity.getCode(), emrClusterDefinitionEntity.getName(), emrClusterAlternateKeyDto.getEmrClusterName(), accountId, emrClusterStatus, emrClusterCreated, request.isDryRun(), emrClusterDefinition);
}
Aggregations