Examples with Tag com.amazonaws.services.autoscaling.model.Tag used on opensource projects

Example 1 with Tag

use of com.amazonaws.services.autoscaling.model.Tag in project incubator-gobblin by apache.

the class GobblinAWSClusterLauncher method getReconnectableClusterId.

@VisibleForTesting
Optional<String> getReconnectableClusterId() throws IOException {
    // List ASGs with Tag of cluster name
    final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
    final List<AutoScalingGroup> autoScalingGroups = this.awsSdkClient.getAutoScalingGroupsWithTag(clusterNameTag);
    // If no auto scaling group is found, we don't have an existing cluster to connect to
    if (autoScalingGroups.size() == 0) {
        return Optional.absent();
    }
    // If more than 0 auto scaling groups are found, validate the setup
    if (autoScalingGroups.size() != 2) {
        throw new IOException("Expected 2 auto scaling groups (1 each for master and workers) but found: " + autoScalingGroups.size());
    }
    // Retrieve cluster information from ASGs
    Optional<String> clusterId = Optional.absent();
    Optional<AutoScalingGroup> masterAsg = Optional.absent();
    Optional<AutoScalingGroup> workersAsg = Optional.absent();
    for (TagDescription tagDescription : autoScalingGroups.get(0).getTags()) {
        LOGGER.info("Found tag: " + tagDescription);
        if (tagDescription.getKey().equalsIgnoreCase(CLUSTER_ID_ASG_TAG)) {
            clusterId = Optional.of(tagDescription.getValue());
        }
        if (tagDescription.getKey().equalsIgnoreCase(ASG_TYPE_ASG_TAG)) {
            if (tagDescription.getValue().equalsIgnoreCase(ASG_TYPE_MASTER)) {
                masterAsg = Optional.of(autoScalingGroups.get(0));
                workersAsg = Optional.of(autoScalingGroups.get(1));
            } else {
                masterAsg = Optional.of(autoScalingGroups.get(1));
                workersAsg = Optional.of(autoScalingGroups.get(0));
            }
        }
    }
    if (!clusterId.isPresent()) {
        throw new IOException("Found 2 auto scaling group names for: " + this.clusterName + " but tags seem to be corrupted, hence could not determine cluster id");
    }
    if (!masterAsg.isPresent() || !workersAsg.isPresent()) {
        throw new IOException("Found 2 auto scaling group names for: " + this.clusterName + " but tags seem to be corrupted, hence could not determine master and workers ASG");
    }
    // Get Master and Workers launch config name and auto scaling group name
    this.masterAutoScalingGroupName = masterAsg.get().getAutoScalingGroupName();
    this.masterLaunchConfigName = masterAsg.get().getLaunchConfigurationName();
    this.workerAutoScalingGroupName = workersAsg.get().getAutoScalingGroupName();
    this.workerLaunchConfigName = workersAsg.get().getLaunchConfigurationName();
    LOGGER.info("Trying to find cluster master public ip");
    this.masterPublicIp = getMasterPublicIp();
    LOGGER.info("Master public ip: " + this.masterPublicIp);
    return clusterId;
}

Example 2 with Tag

use of com.amazonaws.services.autoscaling.model.Tag in project incubator-gobblin by apache.

the class AWSSdkClient method createAutoScalingGroup.

/**
 * Create and launch an {@link AmazonAutoScaling} group
 *
 * @param groupName Auto scaling group name
 * @param launchConfig Launch configuration string
 * @param minSize Minimum number of instances to maintain in auto scaling group
 * @param maxSize Maximum number of instances to scale up-to for load
 * @param desiredCapacity Desired number of instances to maintain in auto scaling group
 * @param availabilityZones Optional availability zones to make use of
 * @param cooldown Optional cooldown period before any scaling event (default is 300 secs)
 * @param healthCheckGracePeriod Optional grace period till which no health check is performed after bootup (default is 300 secs)
 * @param healthCheckType Optional health check type (default is EC2 instance check)
 * @param loadBalancer Optional load balancer to use
 * @param terminationPolicy Optional termination policies
 * @param tags Optional tags to set on auto scaling group (they are set to propagate to EC2 instances implicitly)
 */
public void createAutoScalingGroup(String groupName, String launchConfig, Integer minSize, Integer maxSize, Integer desiredCapacity, Optional<String> availabilityZones, Optional<Integer> cooldown, Optional<Integer> healthCheckGracePeriod, Optional<String> healthCheckType, Optional<String> loadBalancer, Optional<String> terminationPolicy, List<Tag> tags) {
    AmazonAutoScaling autoScaling = getAmazonAutoScalingClient();
    // Propagate ASG tags to EC2 instances launched under the ASG by default
    // (we want to ensure this, hence not configurable)
    final List<Tag> tagsWithPropagationSet = Lists.newArrayList();
    for (Tag tag : tags) {
        tagsWithPropagationSet.add(tag.withPropagateAtLaunch(true));
    }
    CreateAutoScalingGroupRequest createAutoScalingGroupRequest = new CreateAutoScalingGroupRequest().withAutoScalingGroupName(groupName).withLaunchConfigurationName(launchConfig).withMinSize(minSize).withMaxSize(maxSize).withDesiredCapacity(desiredCapacity).withTags(tagsWithPropagationSet);
    if (availabilityZones.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withAvailabilityZones(SPLITTER.splitToList(availabilityZones.get()));
    }
    if (cooldown.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withDefaultCooldown(cooldown.get());
    }
    if (healthCheckGracePeriod.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withHealthCheckGracePeriod(healthCheckGracePeriod.get());
    }
    if (healthCheckType.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withHealthCheckType(healthCheckType.get());
    }
    if (loadBalancer.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withLoadBalancerNames(SPLITTER.splitToList(loadBalancer.get()));
    }
    if (terminationPolicy.isPresent()) {
        createAutoScalingGroupRequest = createAutoScalingGroupRequest.withTerminationPolicies(SPLITTER.splitToList(terminationPolicy.get()));
    }
    autoScaling.createAutoScalingGroup(createAutoScalingGroupRequest);
    LOGGER.info("Created AutoScalingGroup: " + groupName);
}

Example 3 with Tag

use of com.amazonaws.services.autoscaling.model.Tag in project incubator-gobblin by apache.

the class GobblinAWSClusterLauncher method launchClusterMaster.

private String launchClusterMaster(String uuid, String keyName, String securityGroups, AvailabilityZone availabilityZone) {
    // Get cloud-init script to launch cluster master
    final String userData = CloudInitScriptBuilder.buildClusterMasterCommand(this.clusterName, this.nfsParentDir, this.sinkLogRootDir, this.awsConfDir, this.appWorkDir, this.masterS3ConfUri, this.masterS3ConfFiles, this.masterS3JarsUri, this.masterS3JarsFiles, this.masterJarsDir, this.masterJvmMemory, this.masterJvmArgs, this.gobblinVersion);
    // Create launch config for Cluster master
    this.masterLaunchConfigName = MASTER_LAUNCH_CONFIG_NAME_PREFIX + uuid;
    this.awsSdkClient.createLaunchConfig(this.masterLaunchConfigName, this.masterAmiId, this.masterInstanceType, keyName, securityGroups, Optional.<String>absent(), Optional.<String>absent(), Optional.<BlockDeviceMapping>absent(), Optional.<String>absent(), Optional.<InstanceMonitoring>absent(), userData);
    // Create ASG for Cluster master
    // TODO: Make size configurable when we have support multi-master
    this.masterAutoScalingGroupName = MASTER_ASG_NAME_PREFIX + uuid;
    final int minNumMasters = 1;
    final int maxNumMasters = 1;
    final int desiredNumMasters = 1;
    final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
    final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
    final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_MASTER);
    this.awsSdkClient.createAutoScalingGroup(this.masterAutoScalingGroupName, this.masterLaunchConfigName, minNumMasters, maxNumMasters, desiredNumMasters, Optional.of(availabilityZone.getZoneName()), Optional.<Integer>absent(), Optional.<Integer>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));
    LOGGER.info("Waiting for cluster master to launch");
    this.masterPublicIp = getMasterPublicIp();
    LOGGER.info("Master public ip: " + this.masterPublicIp);
    return uuid;
}

Example 4 with Tag

use of com.amazonaws.services.autoscaling.model.Tag in project incubator-gobblin by apache.

the class GobblinAWSClusterLauncher method launchWorkUnitRunners.

private void launchWorkUnitRunners(String uuid, String keyName, String securityGroups, AvailabilityZone availabilityZone) {
    // Get cloud-init script to launch cluster worker
    final String userData = CloudInitScriptBuilder.buildClusterWorkerCommand(this.clusterName, this.nfsParentDir, this.sinkLogRootDir, this.awsConfDir, this.appWorkDir, this.masterPublicIp, this.workerS3ConfUri, this.workerS3ConfFiles, this.workerS3JarsUri, this.workerS3JarsFiles, this.workerJarsDir, this.workerJvmMemory, this.workerJvmArgs, this.gobblinVersion);
    // Create launch config for Cluster worker
    this.workerLaunchConfigName = WORKERS_LAUNCH_CONFIG_PREFIX + uuid;
    this.awsSdkClient.createLaunchConfig(this.workerLaunchConfigName, this.workerAmiId, this.workerInstanceType, keyName, securityGroups, Optional.<String>absent(), Optional.<String>absent(), Optional.<BlockDeviceMapping>absent(), Optional.<String>absent(), Optional.<InstanceMonitoring>absent(), userData);
    // Create ASG for Cluster workers
    this.workerAutoScalingGroupName = WORKERS_ASG_NAME_PREFIX + uuid;
    final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
    final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
    final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_WORKERS);
    this.awsSdkClient.createAutoScalingGroup(this.workerAutoScalingGroupName, this.workerLaunchConfigName, this.minWorkers, this.maxWorkers, this.desiredWorkers, Optional.of(availabilityZone.getZoneName()), Optional.<Integer>absent(), Optional.<Integer>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));
}