Examples with AbstractTopBottomAggregator org.apache.apex.malhar.lib.dimensions.aggregator.AbstractTopBottomAggregator used on opensource projects

Example 1 with AbstractTopBottomAggregator

use of org.apache.apex.malhar.lib.dimensions.aggregator.AbstractTopBottomAggregator in project apex-malhar by apache.

the class DimensionalConfigurationSchema method fulfillCompositeAggregatorExtraInfo.

/**
 * fulfill the embed ddids of composite aggregators
 * get the dimensional descriptor of composite aggregator; genereate field keys of embed aggregator
 */
protected void fulfillCompositeAggregatorExtraInfo() {
    Map<Set<String>, Integer> keysToCombinationId = getKeysToCombinationId();
    final int timeBucketSize = customTimeBuckets.size();
    for (int index = 0; index < dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.size(); ++index) {
        Map<String, FieldsDescriptor> compositeAggregatorNameToDescriptor = dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.get(index);
        for (String compositeAggregatorName : compositeAggregatorNameToDescriptor.keySet()) {
            AbstractTopBottomAggregator compositeAggregator = aggregatorRegistry.getNameToTopBottomAggregator().get(compositeAggregatorName);
            // set DimensionDescriptorID
            compositeAggregator.setDimensionDescriptorID(index);
            // aggregator id
            compositeAggregator.setAggregatorID(aggregatorRegistry.getTopBottomAggregatorNameToID().get(compositeAggregatorName));
            // keys for embed aggregator
            Set<String> keys = Sets.newHashSet();
            DimensionsDescriptor dd = dimensionsDescriptorIDToDimensionsDescriptor.get(index);
            keys.addAll(dd.getFields().getFieldsList());
            {
                Set<String> compositeKeys = Sets.newHashSet();
                compositeKeys.addAll(keys);
                compositeAggregator.setFields(compositeKeys);
                compositeAggregator.setAggregateDescriptor(compositeAggregatorNameToDescriptor.get(compositeAggregatorName));
            }
            keys.addAll(compositeAggregator.getSubCombinations());
            Integer combinationId = keysToCombinationId.get(keys);
            if (combinationId == null) {
                throw new RuntimeException("Can't find combination id for keys: " + keys);
            }
            for (int ddid = combinationId * timeBucketSize; ddid < (combinationId + 1) * timeBucketSize; ++ddid) {
                compositeAggregator.addEmbedAggregatorDdId(ddid);
            }
        }
    }
}

Example 2 with AbstractTopBottomAggregator

use of org.apache.apex.malhar.lib.dimensions.aggregator.AbstractTopBottomAggregator in project apex-malhar by apache.

the class DimensionalConfigurationSchema method addCompositeAggregator.

protected CompositeAggregator addCompositeAggregator(String aggregatorType, Map<String, Set<String>> allValueToCompositeAggregator, Set<String> aggregateCompositeSet, String valueName, String embededAggregatorName, Map<String, Object> properties, Map<String, Type> aggregatorToType) {
    if (!aggregatorRegistry.isTopBottomAggregatorType(aggregatorType)) {
        throw new IllegalArgumentException(aggregatorType + " is not a valid composite aggregator.");
    }
    final String aggregatorName = compositeAggregatorFactory.getCompositeAggregatorName(aggregatorType, embededAggregatorName, properties);
    final CompositeAggregator aggregator = compositeAggregatorFactory.createCompositeAggregator(aggregatorType, embededAggregatorName, properties);
    // Check for duplicate
    Set<String> aggregatorNames = allValueToCompositeAggregator.get(valueName);
    if (aggregatorNames == null) {
        aggregatorNames = Sets.newHashSet();
        allValueToCompositeAggregator.put(valueName, aggregatorNames);
    }
    if (!aggregatorNames.add(aggregatorName)) {
        throw new IllegalArgumentException("An aggregator " + aggregatorName + " cannot be specified twice for value '" + valueName + "'");
    }
    allValueToCompositeAggregator.put(valueName, aggregatorNames);
    aggregateCompositeSet.add(aggregatorName);
    // Add aggregator to the repository
    if (aggregator instanceof AbstractTopBottomAggregator) {
        aggregatorRegistry.getNameToTopBottomAggregator().put(aggregatorName, (AbstractTopBottomAggregator) aggregator);
    }
    LOG.debug("field name {} and adding aggregator names {}:", valueName, aggregatorNames);
    if (aggregatorToType != null) {
        aggregatorToType.put(aggregatorName, aggregator.getOutputType());
    }
    return aggregator;
}

Example 3 with AbstractTopBottomAggregator

use of org.apache.apex.malhar.lib.dimensions.aggregator.AbstractTopBottomAggregator in project apex-malhar by apache.

the class DimensionalConfigurationSchema method computeAdditionalDimensionForCompositeAggregators.

/**
 * The composite aggregators could add additional dimensions and aggregators.
 * This method parse all composite aggregators to genereate additional dimensions and aggregators(embeded
 * aggregators).
 */
protected void computeAdditionalDimensionForCompositeAggregators() {
    // NOTES: dimensionsDescriptorIDToKeys doesn't have time bucket combination while
    // dimensionsDescriptorIDToValueToCompositeAggregator has.
    // the fieldsCombinations generated just check if the keys already existed.
    Map<Set<String>, Integer> keysToCombinationId = getKeysToCombinationId();
    // the dimensionsDescriptorIDToKeys will be change during the process, only to go through the initial for
    // composite aggregator.
    final int initialKeysCombinationsSize = dimensionsDescriptorIDToKeys.size();
    for (int keysIndex = 0; keysIndex < initialKeysCombinationsSize; ++keysIndex) {
        Set<String> keys = dimensionsDescriptorIDToKeys.get(keysIndex).getFields();
        int ddId = keysIndex * customTimeBucketsCombination.size();
        Map<String, Set<String>> valueToAggregators = dimensionsDescriptorIDToValueToCompositeAggregator.get(ddId);
        Map<String, Set<String>> compositeAggregatorToValues = getAggregatorToValues(valueToAggregators);
        // subCombination
        for (Map.Entry<String, Set<String>> aggregatorToValuesEntry : compositeAggregatorToValues.entrySet()) {
            AbstractTopBottomAggregator aggregator = getCompositeAggregatorByName(aggregatorToValuesEntry.getKey());
            Set<String> subCombination = aggregator.getSubCombinations();
            addSubKeysAndAggregator(aggregatorToValuesEntry.getValue(), keys, subCombination, aggregator.getEmbedAggregatorName(), keysToCombinationId);
        }
    }
}