Examples with Resource com.netflix.simianarmy.Resource used on opensource projects

Example 1 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaASGJanitorCrawler method getASGResourcesInRegion.

private List<Resource> getASGResourcesInRegion(String region, String... asgNames) {
    String url = eddaClient.getBaseUrl(region) + "/aws/autoScalingGroups;";
    if (asgNames != null && asgNames.length != 0) {
        url += StringUtils.join(asgNames, ',');
        LOGGER.info(String.format("Getting ASGs in region %s for %d ids", region, asgNames.length));
    } else {
        LOGGER.info(String.format("Getting all ASGs in region %s", region));
    }
    url += ";_expand:(autoScalingGroupName,createdTime,maxSize,suspendedProcesses:(processName,suspensionReason)," + "tags:(key,value),instances:(instanceId),loadBalancerNames,launchConfigurationName)";
    JsonNode jsonNode = null;
    try {
        jsonNode = eddaClient.getJsonNodeFromUrl(url);
    } catch (Exception e) {
        LOGGER.error(String.format("Failed to get Jason node from edda for ASGs in region %s.", region), e);
    }
    if (jsonNode == null || !jsonNode.isArray()) {
        throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
    }
    Map<String, Long> lcNameToCreationTime = getLaunchConfigCreationTimes(region);
    List<Resource> resources = Lists.newArrayList();
    for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
        resources.add(parseJsonElementToresource(region, it.next(), lcNameToCreationTime));
    }
    return resources;
}

Example 2 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaEBSVolumeJanitorCrawler method getUnattachedVolumeResourcesInRegion.

/**
 * Gets all volumes that are not attached to any instance. Janitor Monkey only considers unattached volumes
 * as cleanup candidates, so there is no need to get volumes that are in-use.
 * @param region
 * @return list of resources that are not attached to any instance
 */
private List<Resource> getUnattachedVolumeResourcesInRegion(String region, String... volumeIds) {
    String url = eddaClient.getBaseUrl(region) + "/aws/volumes;";
    if (volumeIds != null && volumeIds.length != 0) {
        url += StringUtils.join(volumeIds, ',');
        LOGGER.info(String.format("Getting volumes in region %s for %d ids", region, volumeIds.length));
    } else {
        LOGGER.info(String.format("Getting all unattached volumes in region %s", region));
    }
    url += ";state=available;_expand:(volumeId,createTime,size,state,tags)";
    JsonNode jsonNode = null;
    try {
        jsonNode = eddaClient.getJsonNodeFromUrl(url);
    } catch (Exception e) {
        LOGGER.error(String.format("Failed to get Jason node from edda for unattached volumes in region %s.", region), e);
    }
    if (jsonNode == null || !jsonNode.isArray()) {
        throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
    }
    List<Resource> resources = Lists.newArrayList();
    for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
        resources.add(parseJsonElementToVolumeResource(region, it.next()));
    }
    return resources;
}

Example 3 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaEBSVolumeJanitorCrawler method addLastAttachmentInfo.

/**
 * Adds information of last attachment to the resources. To be compatible with the AWS implementation of
 * the same crawler, add the information to the JANITOR_META tag. It always uses the latest information
 * to update the tag in this resource (not writing back to AWS) no matter if the tag exists.
 * @param resources the volume resources
 */
private void addLastAttachmentInfo(List<Resource> resources) {
    Validate.notNull(resources);
    LOGGER.info(String.format("Updating the latest attachment info for %d resources", resources.size()));
    Map<String, List<Resource>> regionToResources = Maps.newHashMap();
    for (Resource resource : resources) {
        List<Resource> regionalList = regionToResources.get(resource.getRegion());
        if (regionalList == null) {
            regionalList = Lists.newArrayList();
            regionToResources.put(resource.getRegion(), regionalList);
        }
        regionalList.add(resource);
    }
    for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
        LOGGER.info(String.format("Updating the latest attachment info for %d resources in region %s", resources.size(), entry.getKey()));
        for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
            LOGGER.info(String.format("Processing batch of size %d", batch.size()));
            String batchUrl = getBatchUrl(entry.getKey(), batch);
            JsonNode batchResult = null;
            try {
                batchResult = eddaClient.getJsonNodeFromUrl(batchUrl);
            } catch (IOException e) {
                LOGGER.error("Failed to get response for the batch.", e);
            }
            Map<String, Resource> idToResource = Maps.newHashMap();
            for (Resource resource : batch) {
                idToResource.put(resource.getId(), resource);
            }
            if (batchResult == null || !batchResult.isArray()) {
                throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", batchUrl, batchResult));
            }
            Set<String> processedIds = Sets.newHashSet();
            for (Iterator<JsonNode> it = batchResult.getElements(); it.hasNext(); ) {
                JsonNode elem = it.next();
                JsonNode data = elem.get("data");
                String volumeId = data.get("volumeId").getTextValue();
                Resource resource = idToResource.get(volumeId);
                JsonNode attachments = data.get("attachments");
                if (!(attachments.isArray() && attachments.size() > 0)) {
                    continue;
                }
                JsonNode attachment = attachments.get(0);
                JsonNode ltime = elem.get("ltime");
                if (ltime == null || ltime.isNull()) {
                    continue;
                }
                DateTime detachTime = new DateTime(ltime.asLong());
                processedIds.add(volumeId);
                setAttachmentInfo(volumeId, attachment, detachTime, resource);
            }
            for (Map.Entry<String, Resource> volumeEntry : idToResource.entrySet()) {
                String id = volumeEntry.getKey();
                if (!processedIds.contains(id)) {
                    Resource resource = volumeEntry.getValue();
                    LOGGER.info(String.format("Volume %s never was attached, use createTime %s as the detachTime", id, resource.getLaunchTime()));
                    setAttachmentInfo(id, null, new DateTime(resource.getLaunchTime().getTime()), resource);
                }
            }
        }
    }
}

Example 4 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaEBSVolumeJanitorCrawler method parseJsonElementToVolumeResource.

private Resource parseJsonElementToVolumeResource(String region, JsonNode jsonNode) {
    Validate.notNull(jsonNode);
    long createTime = jsonNode.get("createTime").asLong();
    Resource resource = new AWSResource().withId(jsonNode.get("volumeId").getTextValue()).withRegion(region).withResourceType(AWSResourceType.EBS_VOLUME).withLaunchTime(new Date(createTime));
    JsonNode tags = jsonNode.get("tags");
    StringBuilder description = new StringBuilder();
    JsonNode size = jsonNode.get("size");
    description.append(String.format("size=%s", size == null ? "unknown" : size.getIntValue()));
    if (tags == null || !tags.isArray() || tags.size() == 0) {
        LOGGER.debug(String.format("No tags is found for %s", resource.getId()));
    } else {
        for (Iterator<JsonNode> it = tags.getElements(); it.hasNext(); ) {
            JsonNode tag = it.next();
            String key = tag.get("key").getTextValue();
            String value = tag.get("value").getTextValue();
            description.append(String.format("; %s=%s", key, value));
            resource.setTag(key, value);
            if (key.equals(PURPOSE)) {
                resource.setAdditionalField(PURPOSE, value);
            }
        }
        resource.setDescription(description.toString());
    }
    ((AWSResource) resource).setAWSResourceState(jsonNode.get("state").getTextValue());
    return resource;
}

Example 5 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaELBJanitorCrawler method getELBResourcesInRegion.

private List<Resource> getELBResourcesInRegion(String region, String... elbNames) {
    String url = eddaClient.getBaseUrl(region) + "/aws/loadBalancers";
    if (elbNames != null && elbNames.length != 0) {
        url += StringUtils.join(elbNames, ',');
        LOGGER.info(String.format("Getting ELBs in region %s for %d names", region, elbNames.length));
    } else {
        LOGGER.info(String.format("Getting all ELBs in region %s", region));
    }
    url += ";_expand:(loadBalancerName,createdTime,DNSName,instances,tags:(key,value))";
    JsonNode jsonNode = null;
    try {
        jsonNode = eddaClient.getJsonNodeFromUrl(url);
    } catch (Exception e) {
        LOGGER.error(String.format("Failed to get Jason node from edda for ELBs in region %s.", region), e);
    }
    if (jsonNode == null || !jsonNode.isArray()) {
        throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
    }
    List<Resource> resources = Lists.newArrayList();
    for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
        resources.add(parseJsonElementToELBResource(region, it.next()));
    }
    Map<String, List<String>> elBtoASGMap = buildELBtoASGMap(region);
    for (Resource resource : resources) {
        List<String> asgList = elBtoASGMap.get(resource.getId());
        if (asgList != null && asgList.size() > 0) {
            resource.setAdditionalField("referencedASGCount", "" + asgList.size());
            String asgStr = StringUtils.join(asgList, ",");
            resource.setDescription(resource.getDescription() + ", ASGS=" + asgStr);
            LOGGER.debug(String.format("Resource ELB %s is referenced by ASGs %s", resource.getId(), asgStr));
        } else {
            resource.setAdditionalField("referencedASGCount", "0");
            resource.setDescription(resource.getDescription() + ", ASGS=none");
            LOGGER.debug(String.format("No ASGs found for ELB %s", resource.getId()));
        }
    }
    Map<String, List<DNSEntry>> elBtoDNSMap = buildELBtoDNSMap(region);
    for (Resource resource : resources) {
        List<DNSEntry> dnsEntryList = elBtoDNSMap.get(resource.getAdditionalField("DNSName"));
        if (dnsEntryList != null && dnsEntryList.size() > 0) {
            ArrayList<String> dnsNames = new ArrayList<>();
            ArrayList<String> dnsTypes = new ArrayList<>();
            ArrayList<String> hostedZoneIds = new ArrayList<>();
            for (DNSEntry dnsEntry : dnsEntryList) {
                dnsNames.add(dnsEntry.dnsName);
                dnsTypes.add(dnsEntry.dnsType);
                hostedZoneIds.add(dnsEntry.hostedZoneId);
            }
            resource.setAdditionalField("referencedDNS", StringUtils.join(dnsNames, ","));
            resource.setAdditionalField("referencedDNSTypes", StringUtils.join(dnsTypes, ","));
            resource.setAdditionalField("referencedDNSZones", StringUtils.join(hostedZoneIds, ","));
            resource.setDescription(resource.getDescription() + ", DNS=" + resource.getAdditionalField("referencedDNS"));
            LOGGER.debug(String.format("Resource ELB %s is referenced by DNS %s", resource.getId(), resource.getAdditionalField("referencedDNS")));
        } else {
            resource.setAdditionalField("referencedDNS", "");
            resource.setDescription(resource.getDescription() + ", DNS=none");
            LOGGER.debug(String.format("No DNS found for ELB %s", resource.getId()));
        }
    }
    return resources;
}