Examples with Product com.netflix.ice.tag.Product used on opensource projects

Example 1 with Product

use of com.netflix.ice.tag.Product in project ice by Netflix.

the class BasicManagers method doWork.

private void doWork() {
    logger.info("trying to find new tag group and data managers...");
    Set<Product> products = Sets.newHashSet(this.products);
    Map<Product, BasicTagGroupManager> tagGroupManagers = Maps.newHashMap(this.tagGroupManagers);
    TreeMap<Key, BasicDataManager> costManagers = Maps.newTreeMap(this.costManagers);
    TreeMap<Key, BasicDataManager> usageManagers = Maps.newTreeMap(this.usageManagers);
    Set<Product> newProducts = Sets.newHashSet();
    AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
    for (S3ObjectSummary s3ObjectSummary : s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).getObjectSummaries()) {
        String key = s3ObjectSummary.getKey();
        Product product;
        if (key.endsWith("_all")) {
            product = null;
        } else {
            String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length());
            product = config.productService.getProductByName(name);
        }
        if (!products.contains(product)) {
            products.add(product);
            newProducts.add(product);
        }
    }
    for (Product product : newProducts) {
        tagGroupManagers.put(product, new BasicTagGroupManager(product));
        for (ConsolidateType consolidateType : ConsolidateType.values()) {
            Key key = new Key(product, consolidateType);
            costManagers.put(key, new BasicDataManager(product, consolidateType, true));
            usageManagers.put(key, new BasicDataManager(product, consolidateType, false));
        }
    }
    if (newProducts.size() > 0) {
        this.costManagers = costManagers;
        this.usageManagers = usageManagers;
        this.tagGroupManagers = tagGroupManagers;
        this.products = products;
    }
}

Example 2 with Product

use of com.netflix.ice.tag.Product in project ice by Netflix.

the class BasicProductService method getProductByAwsName.

public Product getProductByAwsName(String awsName) {
    Product product = productsByAwsName.get(awsName);
    if (product == null) {
        product = new Product(awsName);
        productsByAwsName.put(awsName, product);
        productsByName.put(awsName, product);
    }
    return product;
}

Example 3 with Product

use of com.netflix.ice.tag.Product in project ice by Netflix.

the class BillingFileProcessor method archive.

private void archive() throws Exception {
    logger.info("archiving tag data...");
    for (Product product : costDataByProduct.keySet()) {
        TagGroupWriter writer = new TagGroupWriter(product == null ? "all" : product.name);
        writer.archive(startMilli, costDataByProduct.get(product).getTagGroups());
    }
    logger.info("archiving summary data...");
    archiveSummary(usageDataByProduct, "usage_");
    archiveSummary(costDataByProduct, "cost_");
    logger.info("archiving hourly data...");
    archiveHourly(usageDataByProduct, "usage_");
    archiveHourly(costDataByProduct, "cost_");
    logger.info("archiving data done.");
}

Example 4 with Product

use of com.netflix.ice.tag.Product in project ice by Netflix.

the class BillingFileProcessor method archiveSummary.

private void archiveSummary(Map<Product, ReadWriteData> dataMap, String prefix) throws Exception {
    DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC);
    for (Product product : dataMap.keySet()) {
        String prodName = product == null ? "all" : product.name;
        ReadWriteData data = dataMap.get(product);
        Collection<TagGroup> tagGroups = data.getTagGroups();
        // init daily, weekly and monthly
        List<Map<TagGroup, Double>> daily = Lists.newArrayList();
        List<Map<TagGroup, Double>> weekly = Lists.newArrayList();
        List<Map<TagGroup, Double>> monthly = Lists.newArrayList();
        // get last month data
        ReadWriteData lastMonthData = new DataWriter(prefix + "hourly_" + prodName + "_" + AwsUtils.monthDateFormat.print(monthDateTime.minusMonths(1)), true).getData();
        // aggregate to daily, weekly and monthly
        int dayOfWeek = monthDateTime.getDayOfWeek();
        int daysFromLastMonth = dayOfWeek - 1;
        int lastMonthNumHours = monthDateTime.minusMonths(1).dayOfMonth().getMaximumValue() * 24;
        for (int hour = 0 - daysFromLastMonth * 24; hour < data.getNum(); hour++) {
            if (hour < 0) {
                // handle data from last month, add to weekly
                Map<TagGroup, Double> prevData = lastMonthData.getData(lastMonthNumHours + hour);
                for (TagGroup tagGroup : tagGroups) {
                    Double v = prevData.get(tagGroup);
                    if (v != null && v != 0) {
                        addValue(weekly, 0, tagGroup, v);
                    }
                }
            } else {
                // this month, add to weekly, monthly and daily
                Map<TagGroup, Double> map = data.getData(hour);
                for (TagGroup tagGroup : tagGroups) {
                    Double v = map.get(tagGroup);
                    if (v != null && v != 0) {
                        addValue(monthly, 0, tagGroup, v);
                        addValue(daily, hour / 24, tagGroup, v);
                        addValue(weekly, (hour + daysFromLastMonth * 24) / 24 / 7, tagGroup, v);
                    }
                }
            }
        }
        // archive daily
        int year = monthDateTime.getYear();
        DataWriter writer = new DataWriter(prefix + "daily_" + prodName + "_" + year, true);
        ReadWriteData dailyData = writer.getData();
        dailyData.setData(daily, monthDateTime.getDayOfYear() - 1, false);
        writer.archive();
        // archive monthly
        writer = new DataWriter(prefix + "monthly_" + prodName, true);
        ReadWriteData monthlyData = writer.getData();
        monthlyData.setData(monthly, Months.monthsBetween(config.startDate, monthDateTime).getMonths(), false);
        writer.archive();
        // archive weekly
        writer = new DataWriter(prefix + "weekly_" + prodName, true);
        ReadWriteData weeklyData = writer.getData();
        DateTime weekStart = monthDateTime.withDayOfWeek(1);
        int index;
        if (!weekStart.isAfter(config.startDate))
            index = 0;
        else
            index = Weeks.weeksBetween(config.startDate, weekStart).getWeeks() + (config.startDate.dayOfWeek() == weekStart.dayOfWeek() ? 0 : 1);
        weeklyData.setData(weekly, index, true);
        writer.archive();
    }
}

Example 5 with Product

use of com.netflix.ice.tag.Product in project ice by Netflix.

the class BillingFileProcessor method archiveHourly.

private void archiveHourly(Map<Product, ReadWriteData> dataMap, String prefix) throws Exception {
    DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC);
    for (Product product : dataMap.keySet()) {
        String prodName = product == null ? "all" : product.name;
        DataWriter writer = new DataWriter(prefix + "hourly_" + prodName + "_" + AwsUtils.monthDateFormat.print(monthDateTime), false);
        writer.archive(dataMap.get(product));
    }
}