Examples with TupleImpl com.twitter.heron.common.utils.tuple.TupleImpl used on opensource projects

Example 6 with TupleImpl

use of com.twitter.heron.common.utils.tuple.TupleImpl in project incubator-heron by apache.

the class BoltOutputCollectorImpl method admitAckTuple.

private void admitAckTuple(Tuple tuple) {
    Duration latency = Duration.ZERO;
    if (ackEnabled) {
        if (tuple instanceof TupleImpl) {
            TupleImpl tuplImpl = (TupleImpl) tuple;
            HeronTuples.AckTuple.Builder bldr = HeronTuples.AckTuple.newBuilder();
            bldr.setAckedtuple(tuplImpl.getTupleKey());
            long tupleSizeInBytes = 0;
            for (HeronTuples.RootId rt : tuplImpl.getRoots()) {
                bldr.addRoots(rt);
                tupleSizeInBytes += rt.getSerializedSize();
            }
            outputter.addAckTuple(bldr, tupleSizeInBytes);
            latency = Duration.ofNanos(System.nanoTime()).minusNanos(tuplImpl.getCreationTime());
        }
    }
    // Invoke user-defined boltAck task hook
    getPhysicalPlanHelper().getTopologyContext().invokeHookBoltAck(tuple, latency);
    boltMetrics.ackedTuple(tuple.getSourceStreamId(), tuple.getSourceComponent(), latency.toNanos());
}

Example 7 with TupleImpl

use of com.twitter.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitAckTuple.

private void admitAckTuple(Tuple tuple) {
    long latency = 0;
    if (ackEnabled) {
        if (tuple instanceof TupleImpl) {
            TupleImpl tuplImpl = (TupleImpl) tuple;
            HeronTuples.AckTuple.Builder bldr = HeronTuples.AckTuple.newBuilder();
            bldr.setAckedtuple(tuplImpl.getTupleKey());
            long tupleSizeInBytes = 0;
            for (HeronTuples.RootId rt : tuplImpl.getRoots()) {
                bldr.addRoots(rt);
                tupleSizeInBytes += rt.getSerializedSize();
            }
            outputter.addAckTuple(bldr, tupleSizeInBytes);
            latency = System.nanoTime() - tuplImpl.getCreationTime();
        }
    }
    // Invoke user-defined boltAck task hook
    helper.getTopologyContext().invokeHookBoltAck(tuple, latency);
    boltMetrics.ackedTuple(tuple.getSourceStreamId(), tuple.getSourceComponent(), latency);
}

Example 8 with TupleImpl

use of com.twitter.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltInstance method readTuplesAndExecute.

@Override
public void readTuplesAndExecute(Communicator<HeronTuples.HeronTupleSet> inQueue) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    long instanceExecuteBatchTime = systemConfig.getInstanceExecuteBatchTimeMs() * Constants.MILLISECONDS_TO_NANOSECONDS;
    long startOfCycle = System.nanoTime();
    // Read data from in Queues
    while (!inQueue.isEmpty()) {
        HeronTuples.HeronTupleSet tuples = inQueue.poll();
        // Handle the tuples
        if (tuples.hasControl()) {
            throw new RuntimeException("Bolt cannot get acks/fails from other components");
        }
        // Get meta data of tuples
        TopologyAPI.StreamId stream = tuples.getData().getStream();
        int nValues = topologyContext.getComponentOutputFields(stream.getComponentName(), stream.getId()).size();
        // We would reuse the System.nanoTime()
        long currentTime = startOfCycle;
        for (HeronTuples.HeronDataTuple dataTuple : tuples.getData().getTuplesList()) {
            // Create the value list and fill the value
            List<Object> values = new ArrayList<>(nValues);
            for (int i = 0; i < nValues; i++) {
                values.add(serializer.deserialize(dataTuple.getValues(i).toByteArray()));
            }
            // Decode the tuple
            TupleImpl t = new TupleImpl(topologyContext, stream, dataTuple.getKey(), dataTuple.getRootsList(), values, currentTime, false);
            // Delegate to the use defined bolt
            bolt.execute(t);
            // Swap
            long startTime = currentTime;
            currentTime = System.nanoTime();
            long executeLatency = currentTime - startTime;
            // Invoke user-defined execute task hook
            topologyContext.invokeHookBoltExecute(t, executeLatency);
            // Update metrics
            boltMetrics.executeTuple(stream.getId(), stream.getComponentName(), executeLatency);
        }
        // To avoid spending too much time
        if (currentTime - startOfCycle - instanceExecuteBatchTime > 0) {
            break;
        }
    }
}

Example 9 with TupleImpl

use of com.twitter.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

/////////////////////////////////////////////////////////
// Following private methods are internal implementations
/////////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
    if (helper.isTerminatedComponent()) {
        // No need to handle this tuples
        return null;
    }
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = HeronTuples.HeronDataTuple.newBuilder();
    // set the key. This is mostly ignored
    bldr.setKey(0);
    List<Integer> customGroupingTargetTaskIds = null;
    if (!helper.isCustomGroupingEmpty()) {
        // customGroupingTargetTaskIds will be null if this stream is not CustomStreamGrouping
        customGroupingTargetTaskIds = helper.chooseTasksForCustomStreamGrouping(streamId, tuple);
        if (customGroupingTargetTaskIds != null) {
            // It is a CustomStreamGrouping
            bldr.addAllDestTaskIds(customGroupingTargetTaskIds);
        }
    }
    // Invoke user-defined emit task hook
    helper.getTopologyContext().invokeHookEmit(tuple, streamId, customGroupingTargetTaskIds);
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<HeronTuples.RootId>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    long tupleSizeInBytes = 0;
    long startTime = System.nanoTime();
    // Serialize it
    for (Object obj : tuple) {
        byte[] b = serializer.serialize(obj);
        ByteString bstr = ByteString.copyFrom(b);
        bldr.addValues(bstr);
        tupleSizeInBytes += b.length;
    }
    long latency = System.nanoTime() - startTime;
    boltMetrics.serializeDataTuple(streamId, latency);
    // submit to outputter
    outputter.addDataTuple(streamId, bldr, tupleSizeInBytes);
    // Update metrics
    boltMetrics.emittedTuple(streamId);
    // TODO:- remove this after changing the api
    return null;
}

Example 10 with TupleImpl

use of com.twitter.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitAckTuple.

private void admitAckTuple(Tuple tuple) {
    if (tuple instanceof TupleImpl) {
        TupleImpl tuplImpl = (TupleImpl) tuple;
        if (ackEnabled) {
            HeronTuples.AckTuple.Builder bldr = HeronTuples.AckTuple.newBuilder();
            bldr.setAckedtuple(tuplImpl.getTupleKey());
            long tupleSizeInBytes = 0;
            for (HeronTuples.RootId rt : tuplImpl.getRoots()) {
                bldr.addRoots(rt);
                tupleSizeInBytes += rt.getSerializedSize();
            }
            outputter.addAckTuple(bldr, tupleSizeInBytes);
        }
        long latency = System.nanoTime() - tuplImpl.getCreationTime();
        // Invoke user-defined boltAck task hook
        helper.getTopologyContext().invokeHookBoltAck(tuple, latency);
        boltMetrics.ackedTuple(tuple.getSourceStreamId(), tuple.getSourceComponent(), latency);
    }
}