Examples with Aggregator storm.trident.operation.Aggregator used on opensource projects

Example 1 with Aggregator

use of storm.trident.operation.Aggregator in project jstorm by alibaba.

the class GroupedStream method partitionAggregate.

@Override
public IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) {
    Aggregator groupedAgg = new GroupedAggregator(agg, _groupFields, inputFields, functionFields.size());
    Fields allInFields = TridentUtils.fieldsUnion(_groupFields, inputFields);
    Fields allOutFields = TridentUtils.fieldsConcat(_groupFields, functionFields);
    Stream s = _stream.partitionAggregate(allInFields, groupedAgg, allOutFields);
    return new GroupedStream(s, _groupFields);
}

Example 2 with Aggregator

use of storm.trident.operation.Aggregator in project storm by nathanmarz.

the class ChainedAggregatorDeclarer method chainEnd.

public Stream chainEnd() {
    Fields[] inputFields = new Fields[_aggs.size()];
    Aggregator[] aggs = new Aggregator[_aggs.size()];
    int[] outSizes = new int[_aggs.size()];
    List<String> allOutFields = new ArrayList<String>();
    Set<String> allInFields = new HashSet<String>();
    for (int i = 0; i < _aggs.size(); i++) {
        AggSpec spec = _aggs.get(i);
        Fields infields = spec.inFields;
        if (infields == null)
            infields = new Fields();
        Fields outfields = spec.outFields;
        if (outfields == null)
            outfields = new Fields();
        inputFields[i] = infields;
        aggs[i] = spec.agg;
        outSizes[i] = outfields.size();
        allOutFields.addAll(outfields.toList());
        allInFields.addAll(infields.toList());
    }
    if (new HashSet(allOutFields).size() != allOutFields.size()) {
        throw new IllegalArgumentException("Output fields for chained aggregators must be distinct: " + allOutFields.toString());
    }
    Fields inFields = new Fields(new ArrayList<String>(allInFields));
    Fields outFields = new Fields(allOutFields);
    Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
    if (_type != AggType.FULL) {
        _stream = _stream.partitionAggregate(inFields, combined, outFields);
    }
    if (_type != AggType.PARTITION) {
        _stream = _globalScheme.aggPartition(_stream);
        BatchToPartition singleEmit = _globalScheme.singleEmitPartitioner();
        Aggregator toAgg = combined;
        if (singleEmit != null) {
            toAgg = new SingleEmitAggregator(combined, singleEmit);
        }
        // this assumes that inFields and outFields are the same for combineragg
        // assumption also made above
        _stream = _stream.partitionAggregate(inFields, toAgg, outFields);
    }
    return _stream.toStream();
}

Example 3 with Aggregator

use of storm.trident.operation.Aggregator in project jstorm by alibaba.

the class ChainedAggregatorDeclarer method chainEnd.

public Stream chainEnd() {
    Fields[] inputFields = new Fields[_aggs.size()];
    Aggregator[] aggs = new Aggregator[_aggs.size()];
    int[] outSizes = new int[_aggs.size()];
    List<String> allOutFields = new ArrayList<String>();
    Set<String> allInFields = new HashSet<String>();
    for (int i = 0; i < _aggs.size(); i++) {
        AggSpec spec = _aggs.get(i);
        Fields infields = spec.inFields;
        if (infields == null)
            infields = new Fields();
        Fields outfields = spec.outFields;
        if (outfields == null)
            outfields = new Fields();
        inputFields[i] = infields;
        aggs[i] = spec.agg;
        outSizes[i] = outfields.size();
        allOutFields.addAll(outfields.toList());
        allInFields.addAll(infields.toList());
    }
    if (new HashSet(allOutFields).size() != allOutFields.size()) {
        throw new IllegalArgumentException("Output fields for chained aggregators must be distinct: " + allOutFields.toString());
    }
    Fields inFields = new Fields(new ArrayList<String>(allInFields));
    Fields outFields = new Fields(allOutFields);
    Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
    if (_type != AggType.FULL) {
        _stream = _stream.partitionAggregate(inFields, combined, outFields);
    }
    if (_type != AggType.PARTITION) {
        _stream = _globalScheme.aggPartition(_stream);
        BatchToPartition singleEmit = _globalScheme.singleEmitPartitioner();
        Aggregator toAgg = combined;
        if (singleEmit != null) {
            toAgg = new SingleEmitAggregator(combined, singleEmit);
        }
        // this assumes that inFields and outFields are the same for combineragg
        // assumption also made above
        _stream = _stream.partitionAggregate(inFields, toAgg, outFields);
    }
    return _stream.toStream();
}

Example 4 with Aggregator

use of storm.trident.operation.Aggregator in project storm by nathanmarz.

the class GroupedStream method partitionAggregate.

@Override
public IAggregatableStream partitionAggregate(Fields inputFields, Aggregator agg, Fields functionFields) {
    Aggregator groupedAgg = new GroupedAggregator(agg, _groupFields, inputFields, functionFields.size());
    Fields allInFields = TridentUtils.fieldsUnion(_groupFields, inputFields);
    Fields allOutFields = TridentUtils.fieldsConcat(_groupFields, functionFields);
    Stream s = _stream.partitionAggregate(allInFields, groupedAgg, allOutFields);
    return new GroupedStream(s, _groupFields);
}