Examples with TupleImpl org.apache.heron.common.utils.tuple.TupleImpl used on opensource projects

Example 1 with TupleImpl

use of org.apache.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltInstance method handleDataTuple.

private void handleDataTuple(HeronTuples.HeronDataTuple dataTuple, TopologyAPI.StreamId stream, int srcTaskId) {
    long startTime = System.nanoTime();
    List<Object> values = new ArrayList<>();
    for (ByteString b : dataTuple.getValuesList()) {
        values.add(serializer.deserialize(b.toByteArray()));
    }
    // Decode the tuple
    TupleImpl t = new TupleImpl(helper.getTopologyContext(), stream, dataTuple.getKey(), dataTuple.getRootsList(), values, srcTaskId);
    long deserializedTime = System.nanoTime();
    // Delegate to the use defined bolt
    bolt.execute(t);
    Duration executeLatency = Duration.ofNanos(System.nanoTime()).minusNanos(deserializedTime);
    // Invoke user-defined execute task hook
    helper.getTopologyContext().invokeHookBoltExecute(t, executeLatency);
    boltMetrics.deserializeDataTuple(stream.getId(), stream.getComponentName(), deserializedTime - startTime);
    // Update metrics
    boltMetrics.executeTuple(stream.getId(), stream.getComponentName(), executeLatency.toNanos());
}

Example 2 with TupleImpl

use of org.apache.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltInstance method readTuplesAndExecute.

@Override
public void readTuplesAndExecute(Communicator<Message> inQueue) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    Duration instanceExecuteBatchTime = systemConfig.getInstanceExecuteBatchTime();
    long startOfCycle = System.nanoTime();
    // Read data from in Queues
    while (!inQueue.isEmpty() && !waitingForCheckpointSaved) {
        Message msg = inQueue.poll();
        if (msg instanceof CheckpointManager.InitiateStatefulCheckpoint) {
            String checkpointId = ((CheckpointManager.InitiateStatefulCheckpoint) msg).getCheckpointId();
            persistState(checkpointId);
        }
        if (msg instanceof HeronTuples.HeronTupleSet) {
            HeronTuples.HeronTupleSet tuples = (HeronTuples.HeronTupleSet) msg;
            // Handle the tuples
            if (tuples.hasControl()) {
                throw new RuntimeException("Bolt cannot get acks/fails from other components");
            }
            // Get meta data of tuples
            TopologyAPI.StreamId stream = tuples.getData().getStream();
            int nValues = topologyContext.getComponentOutputFields(stream.getComponentName(), stream.getId()).size();
            int sourceTaskId = tuples.getSrcTaskId();
            for (HeronTuples.HeronDataTuple dataTuple : tuples.getData().getTuplesList()) {
                long startTime = System.nanoTime();
                // Create the value list and fill the value
                List<Object> values = new ArrayList<>(nValues);
                for (int i = 0; i < nValues; i++) {
                    values.add(serializer.deserialize(dataTuple.getValues(i).toByteArray()));
                }
                // Decode the tuple
                TupleImpl t = new TupleImpl(topologyContext, stream, dataTuple.getKey(), dataTuple.getRootsList(), values, System.nanoTime(), false, sourceTaskId);
                long deserializedTime = System.nanoTime();
                // Delegate to the use defined bolt
                bolt.execute(t);
                // record the latency of execution
                long executeLatency = Duration.ofNanos(System.nanoTime()).minusNanos(deserializedTime).toNanos();
                // Invoke user-defined execute task hook
                topologyContext.invokeHookBoltExecute(t, Duration.ofNanos(executeLatency));
                // Update metrics
                boltMetrics.deserializeDataTuple(stream.getId(), stream.getComponentName(), deserializedTime - startTime);
                boltMetrics.executeTuple(stream.getId(), stream.getComponentName(), executeLatency);
            }
            // To avoid spending too much time
            long currentTime = System.nanoTime();
            if (currentTime - startOfCycle - instanceExecuteBatchTime.toNanos() > 0) {
                break;
            }
        }
    }
}

Example 3 with TupleImpl

use of org.apache.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

// ///////////////////////////////////////////////////////
// Following private methods are internal implementations
// ///////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple, Integer emitDirectTaskId) {
    if (getPhysicalPlanHelper().isTerminatedComponent()) {
        // No need to handle this tuple
        return null;
    }
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = initTupleBuilder(streamId, tuple, emitDirectTaskId);
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    sendTuple(bldr, streamId, tuple);
    // TODO:- remove this after changing the API
    return null;
}

Example 4 with TupleImpl

use of org.apache.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitAckTuple.

private void admitAckTuple(Tuple tuple) {
    Duration latency = Duration.ZERO;
    if (ackEnabled) {
        if (tuple instanceof TupleImpl) {
            TupleImpl tuplImpl = (TupleImpl) tuple;
            HeronTuples.AckTuple.Builder bldr = HeronTuples.AckTuple.newBuilder();
            bldr.setAckedtuple(tuplImpl.getTupleKey());
            long tupleSizeInBytes = 0;
            for (HeronTuples.RootId rt : tuplImpl.getRoots()) {
                bldr.addRoots(rt);
                tupleSizeInBytes += rt.getSerializedSize();
            }
            outputter.addAckTuple(bldr, tupleSizeInBytes);
            latency = Duration.ofNanos(System.nanoTime()).minusNanos(tuplImpl.getCreationTime());
        }
    }
    // Invoke user-defined boltAck task hook
    getPhysicalPlanHelper().getTopologyContext().invokeHookBoltAck(tuple, latency);
    boltMetrics.ackedTuple(tuple.getSourceStreamId(), tuple.getSourceComponent(), latency.toNanos());
}

Example 5 with TupleImpl

use of org.apache.heron.common.utils.tuple.TupleImpl in project heron by twitter.

the class BoltOutputCollectorImpl method admitFailTuple.

private void admitFailTuple(Tuple tuple) {
    Duration latency = Duration.ZERO;
    if (ackEnabled) {
        if (tuple instanceof TupleImpl) {
            TupleImpl tuplImpl = (TupleImpl) tuple;
            HeronTuples.AckTuple.Builder bldr = HeronTuples.AckTuple.newBuilder();
            bldr.setAckedtuple(tuplImpl.getTupleKey());
            long tupleSizeInBytes = 0;
            for (HeronTuples.RootId rt : tuplImpl.getRoots()) {
                bldr.addRoots(rt);
                tupleSizeInBytes += rt.getSerializedSize();
            }
            outputter.addFailTuple(bldr, tupleSizeInBytes);
            latency = Duration.ofNanos(System.nanoTime()).minusNanos(tuplImpl.getCreationTime());
        }
    }
    // Invoke user-defined boltFail task hook
    getPhysicalPlanHelper().getTopologyContext().invokeHookBoltFail(tuple, latency);
    boltMetrics.failedTuple(tuple.getSourceStreamId(), tuple.getSourceComponent(), latency.toNanos());
}