Examples with Resource com.netflix.simianarmy.Resource used on opensource projects

Example 6 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaEBSVolumeJanitorCrawler method getUnattachedVolumeResourcesInRegion.

/**
     * Gets all volumes that are not attached to any instance. Janitor Monkey only considers unattached volumes
     * as cleanup candidates, so there is no need to get volumes that are in-use.
     * @param region
     * @return list of resources that are not attached to any instance
     */
private List<Resource> getUnattachedVolumeResourcesInRegion(String region, String... volumeIds) {
    String url = eddaClient.getBaseUrl(region) + "/aws/volumes;";
    if (volumeIds != null && volumeIds.length != 0) {
        url += StringUtils.join(volumeIds, ',');
        LOGGER.info(String.format("Getting volumes in region %s for %d ids", region, volumeIds.length));
    } else {
        LOGGER.info(String.format("Getting all unattached volumes in region %s", region));
    }
    url += ";state=available;_expand:(volumeId,createTime,size,state,tags)";
    JsonNode jsonNode = null;
    try {
        jsonNode = eddaClient.getJsonNodeFromUrl(url);
    } catch (Exception e) {
        LOGGER.error(String.format("Failed to get Jason node from edda for unattached volumes in region %s.", region), e);
    }
    if (jsonNode == null || !jsonNode.isArray()) {
        throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
    }
    List<Resource> resources = Lists.newArrayList();
    for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
        resources.add(parseJsonElementToVolumeResource(region, it.next()));
    }
    return resources;
}

Example 7 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaEBSVolumeJanitorCrawler method addLastAttachmentInfo.

/**
     * Adds information of last attachment to the resources. To be compatible with the AWS implementation of
     * the same crawler, add the information to the JANITOR_META tag. It always uses the latest information
     * to update the tag in this resource (not writing back to AWS) no matter if the tag exists.
     * @param resources the volume resources
     */
private void addLastAttachmentInfo(List<Resource> resources) {
    Validate.notNull(resources);
    LOGGER.info(String.format("Updating the latest attachment info for %d resources", resources.size()));
    Map<String, List<Resource>> regionToResources = Maps.newHashMap();
    for (Resource resource : resources) {
        List<Resource> regionalList = regionToResources.get(resource.getRegion());
        if (regionalList == null) {
            regionalList = Lists.newArrayList();
            regionToResources.put(resource.getRegion(), regionalList);
        }
        regionalList.add(resource);
    }
    for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
        LOGGER.info(String.format("Updating the latest attachment info for %d resources in region %s", resources.size(), entry.getKey()));
        for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
            LOGGER.info(String.format("Processing batch of size %d", batch.size()));
            String batchUrl = getBatchUrl(entry.getKey(), batch);
            JsonNode batchResult = null;
            try {
                batchResult = eddaClient.getJsonNodeFromUrl(batchUrl);
            } catch (IOException e) {
                LOGGER.error("Failed to get response for the batch.", e);
            }
            Map<String, Resource> idToResource = Maps.newHashMap();
            for (Resource resource : batch) {
                idToResource.put(resource.getId(), resource);
            }
            if (batchResult == null || !batchResult.isArray()) {
                throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", batchUrl, batchResult));
            }
            Set<String> processedIds = Sets.newHashSet();
            for (Iterator<JsonNode> it = batchResult.getElements(); it.hasNext(); ) {
                JsonNode elem = it.next();
                JsonNode data = elem.get("data");
                String volumeId = data.get("volumeId").getTextValue();
                Resource resource = idToResource.get(volumeId);
                JsonNode attachments = data.get("attachments");
                if (!(attachments.isArray() && attachments.size() > 0)) {
                    continue;
                }
                JsonNode attachment = attachments.get(0);
                JsonNode ltime = elem.get("ltime");
                if (ltime == null || ltime.isNull()) {
                    continue;
                }
                DateTime detachTime = new DateTime(ltime.asLong());
                processedIds.add(volumeId);
                setAttachmentInfo(volumeId, attachment, detachTime, resource);
            }
            for (Map.Entry<String, Resource> volumeEntry : idToResource.entrySet()) {
                String id = volumeEntry.getKey();
                if (!processedIds.contains(id)) {
                    Resource resource = volumeEntry.getValue();
                    LOGGER.info(String.format("Volume %s never was attached, use createTime %s as the detachTime", id, resource.getLaunchTime()));
                    setAttachmentInfo(id, null, new DateTime(resource.getLaunchTime().getTime()), resource);
                }
            }
        }
    }
}

Example 8 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaELBJanitorCrawler method getELBResourcesInRegion.

private List<Resource> getELBResourcesInRegion(String region, String... elbNames) {
    String url = eddaClient.getBaseUrl(region) + "/aws/loadBalancers";
    if (elbNames != null && elbNames.length != 0) {
        url += StringUtils.join(elbNames, ',');
        LOGGER.info(String.format("Getting ELBs in region %s for %d names", region, elbNames.length));
    } else {
        LOGGER.info(String.format("Getting all ELBs in region %s", region));
    }
    url += ";_expand:(loadBalancerName,createdTime,DNSName,instances,tags:(key,value))";
    JsonNode jsonNode = null;
    try {
        jsonNode = eddaClient.getJsonNodeFromUrl(url);
    } catch (Exception e) {
        LOGGER.error(String.format("Failed to get Jason node from edda for ELBs in region %s.", region), e);
    }
    if (jsonNode == null || !jsonNode.isArray()) {
        throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
    }
    List<Resource> resources = Lists.newArrayList();
    for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
        resources.add(parseJsonElementToELBResource(region, it.next()));
    }
    Map<String, List<String>> elBtoASGMap = buildELBtoASGMap(region);
    for (Resource resource : resources) {
        List<String> asgList = elBtoASGMap.get(resource.getId());
        if (asgList != null && asgList.size() > 0) {
            resource.setAdditionalField("referencedASGCount", "" + asgList.size());
            String asgStr = StringUtils.join(asgList, ",");
            resource.setDescription(resource.getDescription() + ", ASGS=" + asgStr);
            LOGGER.debug(String.format("Resource ELB %s is referenced by ASGs %s", resource.getId(), asgStr));
        } else {
            resource.setAdditionalField("referencedASGCount", "0");
            resource.setDescription(resource.getDescription() + ", ASGS=none");
            LOGGER.debug(String.format("No ASGs found for ELB %s", resource.getId()));
        }
    }
    Map<String, List<DNSEntry>> elBtoDNSMap = buildELBtoDNSMap(region);
    for (Resource resource : resources) {
        List<DNSEntry> dnsEntryList = elBtoDNSMap.get(resource.getAdditionalField("DNSName"));
        if (dnsEntryList != null && dnsEntryList.size() > 0) {
            ArrayList<String> dnsNames = new ArrayList<>();
            ArrayList<String> dnsTypes = new ArrayList<>();
            ArrayList<String> hostedZoneIds = new ArrayList<>();
            for (DNSEntry dnsEntry : dnsEntryList) {
                dnsNames.add(dnsEntry.dnsName);
                dnsTypes.add(dnsEntry.dnsType);
                hostedZoneIds.add(dnsEntry.hostedZoneId);
            }
            resource.setAdditionalField("referencedDNS", StringUtils.join(dnsNames, ","));
            resource.setAdditionalField("referencedDNSTypes", StringUtils.join(dnsTypes, ","));
            resource.setAdditionalField("referencedDNSZones", StringUtils.join(hostedZoneIds, ","));
            resource.setDescription(resource.getDescription() + ", DNS=" + resource.getAdditionalField("referencedDNS"));
            LOGGER.debug(String.format("Resource ELB %s is referenced by DNS %s", resource.getId(), resource.getAdditionalField("referencedDNS")));
        } else {
            resource.setAdditionalField("referencedDNS", "");
            resource.setDescription(resource.getDescription() + ", DNS=none");
            LOGGER.debug(String.format("No DNS found for ELB %s", resource.getId()));
        }
    }
    return resources;
}

Example 9 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaELBJanitorCrawler method parseJsonElementToELBResource.

private Resource parseJsonElementToELBResource(String region, JsonNode jsonNode) {
    Validate.notNull(jsonNode);
    String elbName = jsonNode.get("loadBalancerName").getTextValue();
    long launchTime = jsonNode.get("createdTime").getLongValue();
    Resource resource = new AWSResource().withId(elbName).withRegion(region).withResourceType(AWSResourceType.ELB).withLaunchTime(new Date(launchTime));
    String dnsName = jsonNode.get("DNSName").getTextValue();
    resource.setAdditionalField("DNSName", dnsName);
    JsonNode tags = jsonNode.get("tags");
    if (tags == null || !tags.isArray() || tags.size() == 0) {
        LOGGER.debug(String.format("No tags is found for %s", resource.getId()));
    } else {
        for (Iterator<JsonNode> it = tags.getElements(); it.hasNext(); ) {
            JsonNode tag = it.next();
            String key = tag.get("key").getTextValue();
            String value = tag.get("value").getTextValue();
            resource.setTag(key, value);
        }
    }
    String owner = getOwnerEmailForResource(resource);
    if (owner != null) {
        resource.setOwnerEmail(owner);
    }
    LOGGER.debug(String.format("Owner of ELB Resource %s (ELB DNS: %s) is %s", resource.getId(), resource.getAdditionalField("DNSName"), resource.getOwnerEmail()));
    JsonNode instances = jsonNode.get("instances");
    if (instances == null || !instances.isArray() || instances.size() == 0) {
        resource.setAdditionalField("instanceCount", "0");
        resource.setDescription("instances=none");
        LOGGER.debug(String.format("No instances found for ELB %s", resource.getId()));
    } else {
        resource.setAdditionalField("instanceCount", "" + instances.size());
        ArrayList<String> instanceList = new ArrayList<String>(instances.size());
        LOGGER.debug(String.format("Found %d instances for ELB %s", instances.size(), resource.getId()));
        for (Iterator<JsonNode> it = instances.getElements(); it.hasNext(); ) {
            JsonNode instance = it.next();
            String instanceId = instance.get("instanceId").getTextValue();
            instanceList.add(instanceId);
        }
        String instancesStr = StringUtils.join(instanceList, ",");
        resource.setDescription(String.format("instances=%s", instances));
        LOGGER.debug(String.format("Resource ELB %s has instances %s", resource.getId(), instancesStr));
    }
    return resource;
}

Example 10 with Resource

use of com.netflix.simianarmy.Resource in project SimianArmy by Netflix.

the class EddaImageJanitorCrawler method addLastReferenceInfo.

private void addLastReferenceInfo(List<Resource> resources, long since) {
    Validate.notNull(resources);
    LOGGER.info(String.format("Updating the latest reference info for %d images", resources.size()));
    Map<String, List<Resource>> regionToResources = Maps.newHashMap();
    for (Resource resource : resources) {
        List<Resource> regionalList = regionToResources.get(resource.getRegion());
        if (regionalList == null) {
            regionalList = Lists.newArrayList();
            regionToResources.put(resource.getRegion(), regionalList);
        }
        regionalList.add(resource);
    }
    //
    for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
        String region = entry.getKey();
        LOGGER.info(String.format("Updating the latest reference info for %d images in region %s", resources.size(), region));
        for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
            LOGGER.info(String.format("Processing batch of size %d", batch.size()));
            updateReferenceTimeByInstance(region, batch, since);
            updateReferenceTimeByLaunchConfig(region, batch, since);
        }
    }
}