use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class ReduceByKeyAndWindowOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
Map<K, T> reduceMap = new HashMap<>();
Map<K, Integer> windowCountMap = new HashMap<>();
for (Tuple tuple : inputWindow.get()) {
R tup = (R) tuple.getValue(0);
addMap(reduceMap, windowCountMap, tup);
}
long startWindow;
long endWindow;
if (inputWindow.getStartTimestamp() == null) {
startWindow = 0;
} else {
startWindow = inputWindow.getStartTimestamp();
}
if (inputWindow.getEndTimestamp() == null) {
endWindow = 0;
} else {
endWindow = inputWindow.getEndTimestamp();
}
for (K key : reduceMap.keySet()) {
Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
}
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class SmartWatchTopology method main.
public static void main(String[] args) throws Exception {
Builder processingGraphBuilder = Builder.newBuilder();
processingGraphBuilder.newSource(SmartWatchReading::new).setName("incoming-watch-readings").reduceByKeyAndWindow(// Key extractor
reading -> reading.getJoggerId(), // Value extractor
reading -> reading.getFeetRun(), // The time window (1 minute of clock time)
WindowConfig.TumblingTimeWindow(Duration.ofSeconds(10)), // The reduce function (produces a cumulative sum)
(cumulative, incoming) -> cumulative + incoming).setName("reduce-to-total-distance-per-jogger").map(keyWindow -> {
// The per-key result of the previous reduce step
long totalFeetRun = keyWindow.getValue();
// The amount of time elapsed
long startTime = keyWindow.getKey().getWindow().getStartTime();
long endTime = keyWindow.getKey().getWindow().getEndTime();
// Cast to float to use as denominator
long timeLengthMillis = endTime - startTime;
// The feet-per-minute calculation
float feetPerMinute = totalFeetRun / (float) (timeLengthMillis / 1000);
// Reduce to two decimal places
String paceString = new DecimalFormat("#.##").format(feetPerMinute);
// Return a per-jogger average pace
return new KeyValue<>(keyWindow.getKey().getKey(), paceString);
}).setName("calculate-average-speed").consume(kv -> {
String logMessage = String.format("(runner: %s, avgFeetPerMinute: %s)", kv.getKey(), kv.getValue());
LOG.info(logMessage);
});
Config config = Config.defaultConfig();
// Fetches the topology name from the first command-line argument
String topologyName = StreamletUtils.getTopologyName(args);
// Finally, the processing graph and configuration are passed to the Runner, which converts
// the graph into a Heron topology that can be run in a Heron cluster.
new Runner().run(topologyName, config, processingGraphBuilder);
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class ReduceByKeyOperator method execute.
@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
R obj = (R) tuple.getValue(0);
K key = keyExtractor.apply(obj);
T value = valueExtractor.apply(obj);
T newValue;
if (reduceMap.containsKey(key)) {
newValue = reduceFn.apply(reduceMap.get(key), value);
} else {
newValue = value;
}
reduceMap.put(key, newValue);
collector.emit(new Values(new KeyValue<K, T>(key, newValue)));
collector.ack(tuple);
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class GeneralReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.
@SuppressWarnings({ "rawtypes", "unchecked" })
private GeneralReduceByKeyAndWindowOperator<KeyValue<String, Integer>, String, Integer> getReduceByWindowOperator(Integer identity) {
GeneralReduceByKeyAndWindowOperator<KeyValue<String, Integer>, String, Integer> reduceByWindowOperator = new GeneralReduceByKeyAndWindowOperator<>(x -> x.getKey(), identity, (o, o2) -> o + o2.getValue());
reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {
@Override
public void reportError(Throwable error) {
}
@Override
public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
emittedTuples.addAll(tuple);
return null;
}
@Override
public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
}
@Override
public void ack(Tuple input) {
}
@Override
public void fail(Tuple input) {
}
}));
return reduceByWindowOperator;
}
use of org.apache.heron.streamlet.KeyValue in project heron by twitter.
the class GeneralReduceByKeyAndWindowOperatorTest method getTupleWindow.
private TupleWindow getTupleWindow(int nkeys, int count) {
TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
List<Tuple> tuples = new LinkedList<>();
for (int i = 0; i < nkeys; i++) {
for (int j = 0; j < count; ++j) {
Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(new KeyValue<>(String.valueOf(i), j)));
tuples.add(tuple);
}
}
TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
return tupleWindow;
}
Aggregations