Examples with AbstractCompositeAggregator org.apache.apex.malhar.lib.dimensions.aggregator.AbstractCompositeAggregator used on opensource projects

Example 1 with AbstractCompositeAggregator

use of org.apache.apex.malhar.lib.dimensions.aggregator.AbstractCompositeAggregator in project apex-malhar by apache.

the class DimensionalConfigurationSchema method buildDimensionsDescriptorIDAggregatorIDMaps.

/**
 * Precondition: all depended aggregators( for example AVG depended on SUM and COUNT, Composite Aggregators
 * depended on embed aggregators )
 * should already solved. This function will not handle this dependencies.
 */
protected void buildDimensionsDescriptorIDAggregatorIDMaps() {
    dimensionsDescriptorIDToIncrementalAggregatorIDs = Lists.newArrayList();
    dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor = Lists.newArrayList();
    dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor = Lists.newArrayList();
    for (int index = 0; index < dimensionsDescriptorIDToAggregatorToAggregateDescriptor.size(); index++) {
        IntArrayList aggIDList = new IntArrayList();
        Int2ObjectMap<FieldsDescriptor> inputMap = new Int2ObjectOpenHashMap<>();
        Int2ObjectMap<FieldsDescriptor> outputMap = new Int2ObjectOpenHashMap<>();
        dimensionsDescriptorIDToIncrementalAggregatorIDs.add(aggIDList);
        dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor.add(inputMap);
        dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor.add(outputMap);
        for (Map.Entry<String, FieldsDescriptor> entry : dimensionsDescriptorIDToAggregatorToAggregateDescriptor.get(index).entrySet()) {
            buildNonCompositeAggregatorIDMap(entry.getKey(), entry.getValue(), aggIDList, inputMap, outputMap);
        }
    }
    // get the max aggregator id for generating the composite aggregator id
    int maxAggregatorID = getLargestNonCompositeAggregatorID();
    // assign aggregatorID to composite aggregators
    dimensionsDescriptorIDToCompositeAggregatorIDs = Lists.newArrayList();
    for (int index = 0; index < dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.size(); index++) {
        IntArrayList aggIDList = new IntArrayList();
        // NOTE: share same map with incremental aggreator. As the input FD and output FD will be get from aggregatorID,
        // so it should be ok to share same map.
        Int2ObjectMap<FieldsDescriptor> inputMap = dimensionsDescriptorIDToAggregatorIDToInputAggregatorDescriptor.get(index);
        Int2ObjectMap<FieldsDescriptor> outputMap = dimensionsDescriptorIDToAggregatorIDToOutputAggregatorDescriptor.get(index);
        dimensionsDescriptorIDToCompositeAggregatorIDs.add(aggIDList);
        for (Map.Entry<String, FieldsDescriptor> entry : dimensionsDescriptorIDToCompositeAggregatorToAggregateDescriptor.get(index).entrySet()) {
            String aggregatorName = entry.getKey();
            FieldsDescriptor inputDescriptor = entry.getValue();
            AbstractCompositeAggregator compositeAggregator = aggregatorRegistry.getNameToTopBottomAggregator().get(aggregatorName);
            // simple use ++ to assign aggregator id
            int aggregatorID;
            Integer objAggregatorID = aggregatorRegistry.getTopBottomAggregatorNameToID().get(aggregatorName);
            if (objAggregatorID == null) {
                aggregatorID = ++maxAggregatorID;
                aggregatorRegistry.getTopBottomAggregatorNameToID().put(aggregatorName, aggregatorID);
            } else {
                aggregatorID = objAggregatorID;
            }
            aggIDList.add(aggregatorID);
            inputMap.put(aggregatorID, inputDescriptor);
            // buildNonCompositeAggregatorIDMap(getEmbededAggregatorName(aggregatorName), entry.getValue(), aggIDList,
            // inputMap, outputMap);
            outputMap.put(aggregatorID, AggregatorUtils.getOutputFieldsDescriptor(inputDescriptor, compositeAggregator));
        }
    }
}