use of com.amazonaws.services.ec2.model.Placement in project h2o-2 by h2oai.
the class EC2 method resize.
//@formatter:on
/**
* Create or terminate EC2 instances. Uses their Name tag to find existing ones.
*/
public Cloud resize() throws Exception {
AmazonEC2Client ec2 = new AmazonEC2Client(new PersistS3.H2OAWSCredentialsProviderChain());
ec2.setEndpoint("ec2." + region + ".amazonaws.com");
DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
List<Reservation> reservations = describeInstancesResult.getReservations();
List<Instance> instances = new ArrayList<Instance>();
for (Reservation reservation : reservations) {
for (Instance instance : reservation.getInstances()) {
String ip = ip(instance);
if (ip != null) {
String name = null;
if (instance.getTags().size() > 0)
name = instance.getTags().get(0).getValue();
if (NAME.equals(name))
instances.add(instance);
}
}
}
System.out.println("Found " + instances.size() + " EC2 instances for user " + USER);
if (instances.size() > boxes) {
for (int i = 0; i < instances.size() - boxes; i++) {
// TODO terminate?
}
} else if (instances.size() < boxes) {
int launchCount = boxes - instances.size();
System.out.println("Creating " + launchCount + " EC2 instances.");
if (confirm) {
System.out.println("Please confirm [y/n]");
String s = Utils.readConsole();
if (s == null || !s.equalsIgnoreCase("y"))
throw new Exception("Aborted");
}
CreatePlacementGroupRequest group = new CreatePlacementGroupRequest();
group.withGroupName(USER);
group.withStrategy(PlacementStrategy.Cluster);
try {
ec2.createPlacementGroup(group);
} catch (AmazonServiceException ex) {
if (!"InvalidPlacementGroup.Duplicate".equals(ex.getErrorCode()))
throw ex;
}
RunInstancesRequest run = new RunInstancesRequest();
run.withInstanceType(type);
run.withImageId(image);
run.withMinCount(launchCount).withMaxCount(launchCount);
run.withSecurityGroupIds(securityGroup);
Placement placement = new Placement();
placement.setGroupName(USER);
run.withPlacement(placement);
BlockDeviceMapping map = new BlockDeviceMapping();
map.setDeviceName("/dev/sdb");
map.setVirtualName("ephemeral0");
run.withBlockDeviceMappings(map);
run.withUserData(new String(Base64.encodeBase64(cloudConfig.getBytes())));
RunInstancesResult runRes = ec2.runInstances(run);
ArrayList<String> ids = new ArrayList<String>();
for (Instance instance : runRes.getReservation().getInstances()) ids.add(instance.getInstanceId());
List<Instance> created = wait(ec2, ids);
System.out.println("Created " + created.size() + " EC2 instances.");
instances.addAll(created);
}
String[] pub = new String[boxes];
String[] prv = new String[boxes];
for (int i = 0; i < boxes; i++) {
pub[i] = instances.get(i).getPublicIpAddress();
prv[i] = instances.get(i).getPrivateIpAddress();
}
System.out.println("EC2 public IPs: " + Utils.join(' ', pub));
System.out.println("EC2 private IPs: " + Utils.join(' ', prv));
Cloud cloud = new Cloud();
cloud.publicIPs.addAll(Arrays.asList(pub));
cloud.privateIPs.addAll(Arrays.asList(prv));
return cloud;
}
use of com.amazonaws.services.ec2.model.Placement in project photon-model by vmware.
the class TestAWSProvisionTask method testProvision.
// Creates a AWS instance via a provision task.
@Test
public void testProvision() throws Throwable {
initResourcePoolAndComputeHost();
// create a AWS VM compute resoruce
boolean addNonExistingSecurityGroup = true;
this.vmState = createAWSVMResource(this.host, this.computeHost, this.endpointState, this.getClass(), this.currentTestName.getMethodName() + "_vm1", zoneId, regionId, null, /* tagLinks */
this.singleNicSpec, addNonExistingSecurityGroup, this.awsTestContext);
// set placement link
String zoneId = TestAWSSetupUtils.zoneId + avalabilityZoneIdentifier;
ComputeState zoneComputeState = createAWSComputeHost(this.host, this.endpointState, zoneId, regionId, this.isAwsClientMock, this.awsMockEndpointReference, null);
zoneComputeState.id = zoneId;
zoneComputeState = TestUtils.doPatch(this.host, zoneComputeState, ComputeState.class, UriUtils.buildUri(this.host, zoneComputeState.documentSelfLink));
if (this.vmState.customProperties == null) {
this.vmState.customProperties = new HashMap<>();
}
this.vmState.customProperties.put(PLACEMENT_LINK, zoneComputeState.documentSelfLink);
TestUtils.doPatch(this.host, this.vmState, ComputeState.class, UriUtils.buildUri(this.host, this.vmState.documentSelfLink));
// kick off a provision task to do the actual VM creation
ProvisionComputeTaskState provisionTask = new ProvisionComputeTaskService.ProvisionComputeTaskState();
provisionTask.computeLink = this.vmState.documentSelfLink;
provisionTask.isMockRequest = this.isMock;
provisionTask.taskSubStage = ProvisionComputeTaskState.SubStage.CREATING_HOST;
// Wait for default request timeout in minutes for the machine to be powered ON before
// reporting failure to the parent task.
provisionTask.documentExpirationTimeMicros = Utils.getNowMicrosUtc() + TimeUnit.MINUTES.toMicros(AWS_VM_REQUEST_TIMEOUT_MINUTES);
provisionTask.tenantLinks = this.endpointState.tenantLinks;
provisionTask = TestUtils.doPost(this.host, provisionTask, ProvisionComputeTaskState.class, UriUtils.buildUri(this.host, ProvisionComputeTaskService.FACTORY_LINK));
this.host.waitForFinishedTask(ProvisionComputeTaskState.class, provisionTask.documentSelfLink);
// check that the VM has been created
ProvisioningUtils.queryComputeInstances(this.host, 3);
if (!this.isMock) {
ComputeState compute = getCompute(this.host, this.vmState.documentSelfLink);
List<Instance> instances = getAwsInstancesByIds(this.client, this.host, Collections.singletonList(compute.id));
Instance instance = instances.get(0);
assertTags(Collections.emptySet(), instance, this.vmState.name);
assertVmNetworksConfiguration(instance);
assertStorageConfiguration(this.client, instance, compute);
assertEquals(zoneId, instance.getPlacement().getAvailabilityZone());
}
this.host.setTimeoutSeconds(600);
this.host.waitFor("Error waiting for stats with default collection windows", () -> {
try {
this.host.log(Level.INFO, "Issuing stats request for VM with default collection window.");
issueStatsRequest(this.vmState, null);
} catch (Throwable t) {
return false;
}
return true;
});
// store the network links and disk links for removal check later
List<String> resourcesToDelete = new ArrayList<>();
if (this.vmState.diskLinks != null) {
resourcesToDelete.addAll(this.vmState.diskLinks);
}
if (this.vmState.networkInterfaceLinks != null) {
resourcesToDelete.addAll(this.vmState.networkInterfaceLinks);
}
// delete vm
TestAWSSetupUtils.deleteVMs(this.vmState.documentSelfLink, this.isMock, this.host);
// validates the local documents of network links and disk links have been removed
verifyRemovalOfResourceState(this.host, resourcesToDelete);
// create another AWS VM
List<String> instanceIdList = new ArrayList<>();
Set<TagState> tags = createTags(null, "testProvisionKey1", "testProvisionValue1", "testProvisionKey2", "testProvisionValue2");
Set<String> tagLinks = tags.stream().map(t -> t.documentSelfLink).collect(Collectors.toSet());
addNonExistingSecurityGroup = false;
this.vmState = createAWSVMResource(this.host, this.computeHost, this.endpointState, this.getClass(), this.currentTestName.getMethodName() + "_vm2", TestAWSSetupUtils.zoneId, regionId, tagLinks, this.singleNicSpec, addNonExistingSecurityGroup, this.awsTestContext);
TestAWSSetupUtils.provisionMachine(this.host, this.vmState, this.isMock, instanceIdList);
if (!this.isMock) {
ComputeState compute = getCompute(this.host, this.vmState.documentSelfLink);
List<Instance> instances = getAwsInstancesByIds(this.client, this.host, Collections.singletonList(compute.id));
assertTags(tags, instances.get(0), this.vmState.name);
assertVmNetworksConfiguration(instances.get(0));
assertStorageConfiguration(this.client, instances.get(0), compute);
// reach out to AWS and get the current state
TestAWSSetupUtils.getBaseLineInstanceCount(this.host, this.client, null);
}
// delete just the local representation of the resource
TestAWSSetupUtils.deleteVMs(this.vmState.documentSelfLink, this.isMock, this.host, true);
if (!this.isMock) {
try {
TestAWSSetupUtils.getBaseLineInstanceCount(this.host, this.client, null);
} finally {
TestAWSSetupUtils.deleteVMsUsingEC2Client(this.client, this.host, instanceIdList);
deleteSecurityGroupUsingEC2Client(this.client, this.host, this.sgToCleanUp);
}
}
this.vmState = null;
this.sgToCleanUp = null;
}
use of com.amazonaws.services.ec2.model.Placement in project druid by druid-io.
the class EC2AutoScaler method provision.
@Override
public AutoScalingData provision() {
try {
final EC2NodeData workerConfig = envConfig.getNodeData();
final String userDataBase64;
if (envConfig.getUserData() == null) {
userDataBase64 = null;
} else {
if (config.getWorkerVersion() == null) {
userDataBase64 = envConfig.getUserData().getUserDataBase64();
} else {
userDataBase64 = envConfig.getUserData().withVersion(config.getWorkerVersion()).getUserDataBase64();
}
}
RunInstancesRequest request = new RunInstancesRequest(workerConfig.getAmiId(), workerConfig.getMinInstances(), workerConfig.getMaxInstances()).withInstanceType(workerConfig.getInstanceType()).withPlacement(new Placement(envConfig.getAvailabilityZone())).withKeyName(workerConfig.getKeyName()).withIamInstanceProfile(workerConfig.getIamProfile() == null ? null : workerConfig.getIamProfile().toIamInstanceProfileSpecification()).withUserData(userDataBase64);
// leaving it null uses the EC2 default.
if (workerConfig.getAssociatePublicIpAddress() != null) {
request.withNetworkInterfaces(new InstanceNetworkInterfaceSpecification().withAssociatePublicIpAddress(workerConfig.getAssociatePublicIpAddress()).withSubnetId(workerConfig.getSubnetId()).withGroups(workerConfig.getSecurityGroupIds()).withDeviceIndex(0));
} else {
request.withSecurityGroupIds(workerConfig.getSecurityGroupIds()).withSubnetId(workerConfig.getSubnetId());
}
final RunInstancesResult result = amazonEC2Client.runInstances(request);
final List<String> instanceIds = Lists.transform(result.getReservation().getInstances(), new Function<Instance, String>() {
@Override
public String apply(Instance input) {
return input.getInstanceId();
}
});
log.info("Created instances: %s", instanceIds);
return new AutoScalingData(Lists.transform(result.getReservation().getInstances(), new Function<Instance, String>() {
@Override
public String apply(Instance input) {
return input.getInstanceId();
}
}));
} catch (Exception e) {
log.error(e, "Unable to provision any EC2 instances.");
}
return null;
}
use of com.amazonaws.services.ec2.model.Placement in project camel by apache.
the class EC2Producer method createAndRunInstance.
private void createAndRunInstance(AmazonEC2Client ec2Client, Exchange exchange) {
String ami;
InstanceType instanceType;
int minCount;
int maxCount;
boolean monitoring;
String kernelId;
boolean ebsOptimized;
Collection securityGroups;
String keyName;
String clientToken;
Placement placement;
RunInstancesRequest request = new RunInstancesRequest();
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.IMAGE_ID))) {
ami = exchange.getIn().getHeader(EC2Constants.IMAGE_ID, String.class);
request.withImageId(ami);
} else {
throw new IllegalArgumentException("AMI must be specified");
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_TYPE))) {
instanceType = exchange.getIn().getHeader(EC2Constants.INSTANCE_TYPE, InstanceType.class);
request.withInstanceType(instanceType.toString());
} else {
throw new IllegalArgumentException("Instance Type must be specified");
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_MIN_COUNT))) {
minCount = exchange.getIn().getHeader(EC2Constants.INSTANCE_MIN_COUNT, Integer.class);
request.withMinCount(minCount);
} else {
throw new IllegalArgumentException("Min instances count must be specified");
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_MAX_COUNT))) {
maxCount = exchange.getIn().getHeader(EC2Constants.INSTANCE_MAX_COUNT, Integer.class);
request.withMaxCount(maxCount);
} else {
throw new IllegalArgumentException("Max instances count must be specified");
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_MONITORING))) {
monitoring = exchange.getIn().getHeader(EC2Constants.INSTANCE_MONITORING, Boolean.class);
request.withMonitoring(monitoring);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_KERNEL_ID))) {
kernelId = exchange.getIn().getHeader(EC2Constants.INSTANCE_KERNEL_ID, String.class);
request.withKernelId(kernelId);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_EBS_OPTIMIZED))) {
ebsOptimized = exchange.getIn().getHeader(EC2Constants.INSTANCE_EBS_OPTIMIZED, Boolean.class);
request.withEbsOptimized(ebsOptimized);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCE_SECURITY_GROUPS))) {
securityGroups = exchange.getIn().getHeader(EC2Constants.INSTANCE_SECURITY_GROUPS, Collection.class);
request.withSecurityGroups(securityGroups);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCES_KEY_PAIR))) {
keyName = exchange.getIn().getHeader(EC2Constants.INSTANCES_KEY_PAIR, String.class);
request.withKeyName(keyName);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCES_CLIENT_TOKEN))) {
clientToken = exchange.getIn().getHeader(EC2Constants.INSTANCES_CLIENT_TOKEN, String.class);
request.withClientToken(clientToken);
}
if (ObjectHelper.isNotEmpty(exchange.getIn().getHeader(EC2Constants.INSTANCES_PLACEMENT))) {
placement = exchange.getIn().getHeader(EC2Constants.INSTANCES_PLACEMENT, Placement.class);
request.withPlacement(placement);
}
RunInstancesResult result;
try {
result = ec2Client.runInstances(request);
} catch (AmazonServiceException ase) {
LOG.trace("Run Instances command returned the error code {}", ase.getErrorCode());
throw ase;
}
LOG.trace("Creating and running instances with ami [{}] and instance type {}", ami, instanceType.toString());
Message message = getMessageForResponse(exchange);
message.setBody(result);
}
use of com.amazonaws.services.ec2.model.Placement in project photon-model by vmware.
the class AWSInstanceService method createInstance.
private void createInstance(AWSInstanceContext aws) {
if (aws.computeRequest.isMockRequest) {
aws.taskManager.finishTask();
return;
}
final DiskState bootDisk = aws.bootDisk;
if (bootDisk == null) {
aws.taskManager.patchTaskToFailure(new IllegalStateException("AWS bootDisk not specified"));
return;
}
if (bootDisk.bootConfig != null && bootDisk.bootConfig.files.length > 1) {
aws.taskManager.patchTaskToFailure(new IllegalStateException("Only 1 configuration file allowed"));
return;
}
// This a single disk state with a bootConfig. There's no expectation
// that it does exists, but if it does, we only support cloud configs at
// this point.
String cloudConfig = null;
if (bootDisk.bootConfig != null && bootDisk.bootConfig.files.length > CLOUD_CONFIG_DEFAULT_FILE_INDEX) {
cloudConfig = bootDisk.bootConfig.files[CLOUD_CONFIG_DEFAULT_FILE_INDEX].contents;
}
String instanceType = aws.child.description.instanceType;
if (instanceType == null) {
// fallback to legacy usage of name
instanceType = aws.child.description.name;
}
if (instanceType == null) {
aws.error = new IllegalStateException("AWS Instance type not specified");
aws.stage = AWSInstanceStage.ERROR;
handleAllocation(aws);
return;
}
RunInstancesRequest runInstancesRequest = new RunInstancesRequest().withImageId(aws.bootDiskImageNativeId).withInstanceType(instanceType).withMinCount(1).withMaxCount(1).withMonitoring(true).withTagSpecifications(new TagSpecification().withResourceType(ResourceType.Instance).withTags(aws.getAWSTags()));
if (aws.placement != null) {
runInstancesRequest.withPlacement(new Placement(aws.placement));
}
if (aws.child.customProperties != null && aws.child.customProperties.containsKey(CUSTOM_PROP_SSH_KEY_NAME)) {
runInstancesRequest = runInstancesRequest.withKeyName(aws.child.customProperties.get(CUSTOM_PROP_SSH_KEY_NAME));
}
if (!aws.dataDisks.isEmpty() || bootDisk.capacityMBytes > 0 || bootDisk.customProperties != null) {
DescribeImagesRequest imagesDescriptionRequest = new DescribeImagesRequest();
imagesDescriptionRequest.withImageIds(aws.bootDiskImageNativeId);
DescribeImagesResult imagesDescriptionResult = aws.amazonEC2Client.describeImages(imagesDescriptionRequest);
if (imagesDescriptionResult.getImages().size() != 1) {
handleError(aws, new IllegalStateException("AWS ImageId is not available"));
return;
}
Image image = imagesDescriptionResult.getImages().get(0);
AssertUtil.assertNotNull(aws.instanceTypeInfo, "instanceType cannot be null");
List<BlockDeviceMapping> blockDeviceMappings = image.getBlockDeviceMappings();
String rootDeviceType = image.getRootDeviceType();
String bootDiskType = bootDisk.customProperties.get(DEVICE_TYPE);
boolean hasHardConstraint = containsHardConstraint(bootDisk);
BlockDeviceMapping rootDeviceMapping = null;
try {
// The number of instance-store disks that will be provisioned is limited by the instance-type.
suppressExcessInstanceStoreDevices(blockDeviceMappings, aws.instanceTypeInfo);
for (BlockDeviceMapping blockDeviceMapping : blockDeviceMappings) {
EbsBlockDevice ebs = blockDeviceMapping.getEbs();
String diskType = getDeviceType(ebs);
if (hasHardConstraint) {
validateIfDeviceTypesAreMatching(diskType, bootDiskType);
}
if (blockDeviceMapping.getNoDevice() != null) {
continue;
}
if (rootDeviceType.equals(AWSStorageType.EBS.getName()) && blockDeviceMapping.getDeviceName().equals(image.getRootDeviceName())) {
rootDeviceMapping = blockDeviceMapping;
continue;
}
DiskState diskState = new DiskState();
copyCustomProperties(diskState, bootDisk);
addMandatoryProperties(diskState, blockDeviceMapping, aws);
updateDeviceMapping(diskType, bootDiskType, blockDeviceMapping.getDeviceName(), ebs, diskState);
// update disk state with final volume-type and iops
if (diskType.equals(AWSStorageType.EBS.getName())) {
diskState.customProperties.put(VOLUME_TYPE, ebs.getVolumeType());
diskState.customProperties.put(DISK_IOPS, String.valueOf(ebs.getIops()));
}
aws.imageDisks.add(diskState);
}
customizeBootDiskProperties(bootDisk, rootDeviceType, rootDeviceMapping, hasHardConstraint, aws);
List<DiskState> ebsDisks = new ArrayList<>();
List<DiskState> instanceStoreDisks = new ArrayList<>();
if (!aws.dataDisks.isEmpty()) {
if (!rootDeviceType.equals(AWSStorageType.EBS.name().toLowerCase())) {
instanceStoreDisks = aws.dataDisks;
assertAndResetPersistence(instanceStoreDisks);
validateSupportForAdditionalInstanceStoreDisks(instanceStoreDisks, blockDeviceMappings, aws.instanceTypeInfo, rootDeviceType);
} else {
splitDataDisks(aws.dataDisks, instanceStoreDisks, ebsDisks);
setEbsDefaultsIfNotSpecified(ebsDisks, Boolean.FALSE);
if (!instanceStoreDisks.isEmpty()) {
assertAndResetPersistence(instanceStoreDisks);
validateSupportForAdditionalInstanceStoreDisks(instanceStoreDisks, blockDeviceMappings, aws.instanceTypeInfo, rootDeviceType);
}
}
}
// get the available attach paths for new disks and external disks
List<String> usedDeviceNames = null;
if (!instanceStoreDisks.isEmpty() || !ebsDisks.isEmpty() || !aws.externalDisks.isEmpty()) {
usedDeviceNames = getUsedDeviceNames(blockDeviceMappings);
}
if (!instanceStoreDisks.isEmpty()) {
List<String> usedVirtualNames = getUsedVirtualNames(blockDeviceMappings);
blockDeviceMappings.addAll(createInstanceStoreMappings(instanceStoreDisks, usedDeviceNames, usedVirtualNames, aws.instanceTypeInfo.id, aws.instanceTypeInfo.dataDiskSizeInMB, image.getPlatform(), image.getVirtualizationType()));
}
if (!ebsDisks.isEmpty() || !aws.externalDisks.isEmpty()) {
aws.availableEbsDiskNames = AWSBlockDeviceNameMapper.getAvailableNames(AWSSupportedOS.get(image.getPlatform()), AWSSupportedVirtualizationTypes.get(image.getVirtualizationType()), AWSStorageType.EBS, instanceType, usedDeviceNames);
}
if (!ebsDisks.isEmpty()) {
blockDeviceMappings.addAll(createEbsDeviceMappings(ebsDisks, aws.availableEbsDiskNames));
}
runInstancesRequest.withBlockDeviceMappings(blockDeviceMappings);
} catch (Exception e) {
aws.error = e;
aws.stage = AWSInstanceStage.ERROR;
handleAllocation(aws);
return;
}
}
AWSNicContext primaryNic = aws.getPrimaryNic();
if (primaryNic != null && primaryNic.nicSpec != null) {
runInstancesRequest.withNetworkInterfaces(primaryNic.nicSpec);
} else {
runInstancesRequest.withSecurityGroupIds(AWSUtils.getOrCreateSecurityGroups(aws, null));
}
if (cloudConfig != null) {
try {
runInstancesRequest.setUserData(Base64.getEncoder().encodeToString(cloudConfig.getBytes(Utils.CHARSET)));
} catch (UnsupportedEncodingException e) {
handleError(aws, new IllegalStateException("Error encoding user data"));
return;
}
}
String message = "[AWSInstanceService] Sending run instance request for instance id: " + aws.bootDiskImageNativeId + ", instance type: " + instanceType + ", parent task id: " + aws.computeRequest.taskReference;
this.logInfo(() -> message);
// handler invoked once the EC2 runInstancesAsync commands completes
AsyncHandler<RunInstancesRequest, RunInstancesResult> creationHandler = new AWSCreationHandler(this, aws);
aws.amazonEC2Client.runInstancesAsync(runInstancesRequest, creationHandler);
}
Aggregations