Examples with Tuple com.twitter.heron.api.tuple.Tuple used on opensource projects

Example 6 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class BoltOutputCollectorImpl method admitBoltTuple.

// ///////////////////////////////////////////////////////
// Following private methods are internal implementations
// ///////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple, Integer emitDirectTaskId) {
    if (getPhysicalPlanHelper().isTerminatedComponent()) {
        // No need to handle this tuple
        return null;
    }
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = initTupleBuilder(streamId, tuple, emitDirectTaskId);
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    sendTuple(bldr, streamId, tuple);
    // TODO:- remove this after changing the api
    return null;
}

Example 7 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class SlidingWindowSumBolt method execute.

@Override
public void execute(TupleWindow inputWindow) {
    /*
             * The inputWindow gives a view of
             * (a) all the events in the window
             * (b) events that expired since last activation of the window
             * (c) events that newly arrived since last activation of the window
             */
    List<Tuple> tuplesInWindow = inputWindow.get();
    List<Tuple> newTuples = inputWindow.getNew();
    List<Tuple> expiredTuples = inputWindow.getExpired();
    LOG.fine("Events in current window: " + tuplesInWindow.size());
    /*
             * Instead of iterating over all the tuples in the window to compute
             * the sum, the values for the new events are added and old events are
             * subtracted. Similar optimizations might be possible in other
             * windowing computations.
             */
    for (Tuple tuple : newTuples) {
        sum += (int) tuple.getValue(0);
    }
    for (Tuple tuple : expiredTuples) {
        sum -= (int) tuple.getValue(0);
    }
    collector.emit(new Values(sum));
}

Example 8 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class WindowedBoltExecutor method initWindowManager.

@SuppressWarnings("unchecked")
private WindowManager<Tuple> initWindowManager(WindowLifecycleListener<Tuple> lifecycleListener, Map<String, Object> topoConf, TopologyContext context, Collection<Event<Tuple>> queue) {
    WindowManager<Tuple> manager = new WindowManager<>(lifecycleListener, queue);
    Count windowLengthCount = null;
    Long slidingIntervalDurationMs = null;
    Count slidingIntervalCount = null;
    // window length
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)) {
        windowLengthCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS)) {
        windowLengthDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS);
    }
    // sliding interval
    if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)) {
        slidingIntervalCount = new Count(((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_COUNT)).intValue());
    } else if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS)) {
        slidingIntervalDurationMs = (Long) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS);
    } else {
        // default is a sliding window of count 1
        slidingIntervalCount = new Count(1);
    }
    // tuple ts
    if (timestampExtractor != null) {
        // late tuple stream
        lateTupleStream = (String) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM);
        if (lateTupleStream != null) {
            if (!context.getThisStreams().contains(lateTupleStream)) {
                throw new IllegalArgumentException("Stream for late tuples must be defined with the " + "builder method withLateTupleStream");
            }
        }
        // max lag
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)) {
            maxLagMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS)).intValue();
        } else {
            maxLagMs = DEFAULT_MAX_LAG_MS;
        }
        // watermark interval
        long watermarkIntervalMs;
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)) {
            watermarkIntervalMs = ((Number) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS)).intValue();
        } else {
            watermarkIntervalMs = DEFAULT_WATERMARK_EVENT_INTERVAL_MS;
        }
        waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkIntervalMs, maxLagMs, getComponentStreams(context), topoConf);
    } else {
        if (topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM)) {
            throw new IllegalArgumentException("Late tuple stream can be defined only when " + "specifying" + " a timestamp field");
        }
    }
    boolean hasCustomTrigger = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
    boolean hasCustomEvictor = topoConf.containsKey(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    if (hasCustomTrigger && hasCustomEvictor) {
        triggerPolicy = (TriggerPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_TRIGGER);
        evictionPolicy = (EvictionPolicy<Tuple, ?>) topoConf.get(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_CUSTOM_EVICTOR);
    } else if (!hasCustomEvictor && !hasCustomTrigger) {
        // validate
        validate(topoConf, windowLengthCount, windowLengthDurationMs, slidingIntervalCount, slidingIntervalDurationMs);
        evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDurationMs);
        triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDurationMs);
    } else {
        throw new IllegalArgumentException("If either a custom TriggerPolicy or EvictionPolicy is defined, both must be.");
    }
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(topoConf);
    triggerPolicy.setTriggerHandler(manager);
    triggerPolicy.setWindowManager(manager);
    manager.setEvictionPolicy(evictionPolicy);
    manager.setTriggerPolicy(triggerPolicy);
    // restore state if there is existing state
    if (this.state != null && this.state.get(WINDOWING_INTERNAL_STATE) != null && !((HashMapState) this.state.get(WINDOWING_INTERNAL_STATE)).isEmpty()) {
        manager.restoreState((Map<String, Serializable>) state.get(WINDOWING_INTERNAL_STATE));
    }
    return manager;
}

Example 9 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class ReduceByKeyAndWindowOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow(int nkeys, int count) {
    TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < nkeys; i++) {
        for (int j = 0; j < count; ++j) {
            Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(String.valueOf(i)));
            tuples.add(tuple);
        }
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}

Example 10 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

/////////////////////////////////////////////////////////
// Following private methods are internal implementations
/////////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
    if (helper.isTerminatedComponent()) {
        // No need to handle this tuples
        return null;
    }
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = HeronTuples.HeronDataTuple.newBuilder();
    // set the key. This is mostly ignored
    bldr.setKey(0);
    List<Integer> customGroupingTargetTaskIds = null;
    if (!helper.isCustomGroupingEmpty()) {
        // customGroupingTargetTaskIds will be null if this stream is not CustomStreamGrouping
        customGroupingTargetTaskIds = helper.chooseTasksForCustomStreamGrouping(streamId, tuple);
        if (customGroupingTargetTaskIds != null) {
            // It is a CustomStreamGrouping
            bldr.addAllDestTaskIds(customGroupingTargetTaskIds);
        }
    }
    // Invoke user-defined emit task hook
    helper.getTopologyContext().invokeHookEmit(tuple, streamId, customGroupingTargetTaskIds);
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<HeronTuples.RootId>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    long tupleSizeInBytes = 0;
    long startTime = System.nanoTime();
    // Serialize it
    for (Object obj : tuple) {
        byte[] b = serializer.serialize(obj);
        ByteString bstr = ByteString.copyFrom(b);
        bldr.addValues(bstr);
        tupleSizeInBytes += b.length;
    }
    long latency = System.nanoTime() - startTime;
    boltMetrics.serializeDataTuple(streamId, latency);
    // submit to outputter
    outputter.addDataTuple(streamId, bldr, tupleSizeInBytes);
    // Update metrics
    boltMetrics.emittedTuple(streamId);
    // TODO:- remove this after changing the api
    return null;
}