Examples with TopologyContextImpl org.apache.heron.common.utils.topology.TopologyContextImpl used on opensource projects

Example 1 with TopologyContextImpl

use of org.apache.heron.common.utils.topology.TopologyContextImpl in project heron by twitter.

the class BoltInstance method readTuplesAndExecute.

@Override
public void readTuplesAndExecute(Communicator<Message> inQueue) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    Duration instanceExecuteBatchTime = systemConfig.getInstanceExecuteBatchTime();
    long startOfCycle = System.nanoTime();
    // Read data from in Queues
    while (!inQueue.isEmpty() && !waitingForCheckpointSaved) {
        Message msg = inQueue.poll();
        if (msg instanceof CheckpointManager.InitiateStatefulCheckpoint) {
            String checkpointId = ((CheckpointManager.InitiateStatefulCheckpoint) msg).getCheckpointId();
            persistState(checkpointId);
        }
        if (msg instanceof HeronTuples.HeronTupleSet) {
            HeronTuples.HeronTupleSet tuples = (HeronTuples.HeronTupleSet) msg;
            // Handle the tuples
            if (tuples.hasControl()) {
                throw new RuntimeException("Bolt cannot get acks/fails from other components");
            }
            // Get meta data of tuples
            TopologyAPI.StreamId stream = tuples.getData().getStream();
            int nValues = topologyContext.getComponentOutputFields(stream.getComponentName(), stream.getId()).size();
            int sourceTaskId = tuples.getSrcTaskId();
            for (HeronTuples.HeronDataTuple dataTuple : tuples.getData().getTuplesList()) {
                long startTime = System.nanoTime();
                // Create the value list and fill the value
                List<Object> values = new ArrayList<>(nValues);
                for (int i = 0; i < nValues; i++) {
                    values.add(serializer.deserialize(dataTuple.getValues(i).toByteArray()));
                }
                // Decode the tuple
                TupleImpl t = new TupleImpl(topologyContext, stream, dataTuple.getKey(), dataTuple.getRootsList(), values, System.nanoTime(), false, sourceTaskId);
                long deserializedTime = System.nanoTime();
                // Delegate to the use defined bolt
                bolt.execute(t);
                // record the latency of execution
                long executeLatency = Duration.ofNanos(System.nanoTime()).minusNanos(deserializedTime).toNanos();
                // Invoke user-defined execute task hook
                topologyContext.invokeHookBoltExecute(t, Duration.ofNanos(executeLatency));
                // Update metrics
                boltMetrics.deserializeDataTuple(stream.getId(), stream.getComponentName(), deserializedTime - startTime);
                boltMetrics.executeTuple(stream.getId(), stream.getComponentName(), executeLatency);
            }
            // To avoid spending too much time
            long currentTime = System.nanoTime();
            if (currentTime - startOfCycle - instanceExecuteBatchTime.toNanos() > 0) {
                break;
            }
        }
    }
}

Example 2 with TopologyContextImpl

use of org.apache.heron.common.utils.topology.TopologyContextImpl in project heron by twitter.

the class SpoutInstance method init.

@SuppressWarnings("unchecked")
@Override
public void init(State<Serializable, Serializable> state) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    // Initialize the GlobalMetrics
    GlobalMetrics.init(topologyContext, systemConfig.getHeronMetricsExportInterval());
    spoutMetrics.registerMetrics(topologyContext);
    // Initialize the instanceState if the topology is stateful and spout is a stateful component
    if (isTopologyStateful && spout instanceof IStatefulComponent) {
        this.instanceState = state;
        ((IStatefulComponent<Serializable, Serializable>) spout).initState(instanceState);
        if (spillState) {
            if (FileUtils.isDirectoryExists(spillStateLocation)) {
                FileUtils.cleanDir(spillStateLocation);
            } else {
                FileUtils.createDirectory(spillStateLocation);
            }
        }
    }
    spout.open(topologyContext.getTopologyConfig(), topologyContext, new SpoutOutputCollector(collector));
    // Invoke user-defined prepare task hook
    topologyContext.invokeHookPrepare();
    // Init the CustomStreamGrouping
    helper.prepareForCustomStreamGrouping();
}

Example 3 with TopologyContextImpl

use of org.apache.heron.common.utils.topology.TopologyContextImpl in project heron by twitter.

the class GlobalMetricsTest method testGlobalMetrics.

@Test
public void testGlobalMetrics() {
    MetricsCollector fakeCollector = new MetricsCollector(new FakeWakeableLooper(), null);
    TopologyContext fakeContext = new TopologyContextImpl(new HashMap<String, Object>(), TopologyAPI.Topology.getDefaultInstance(), new HashMap<Integer, String>(), 0, fakeCollector);
    GlobalMetrics.init(fakeContext, Duration.ofSeconds(5));
    GlobalMetrics.incr("mycounter");
    Map<String, Long> metricsContent = GlobalMetrics.getUnderlyingCounter().getValueAndReset();
    assertTrue(metricsContent.containsKey("mycounter"));
    assertEquals(1, metricsContent.size());
    assertEquals(new Long(1), metricsContent.get("mycounter"));
    // Increment two different counters
    GlobalMetrics.incr("mycounter1");
    GlobalMetrics.incr("mycounter2");
    GlobalMetrics.incr("mycounter1");
    metricsContent = GlobalMetrics.getUnderlyingCounter().getValueAndReset();
    assertTrue(metricsContent.containsKey("mycounter"));
    assertTrue(metricsContent.containsKey("mycounter1"));
    assertTrue(metricsContent.containsKey("mycounter2"));
    assertEquals(3L, metricsContent.size());
    assertEquals(new Long(0), metricsContent.get("mycounter"));
    assertEquals(new Long(1), metricsContent.get("mycounter2"));
    assertEquals(new Long(2), metricsContent.get("mycounter1"));
}

Example 4 with TopologyContextImpl

use of org.apache.heron.common.utils.topology.TopologyContextImpl in project heron by twitter.

the class BoltInstance method init.

@SuppressWarnings("unchecked")
@Override
public void init(State<Serializable, Serializable> state) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    // Initialize the GlobalMetrics
    GlobalMetrics.init(topologyContext, systemConfig.getHeronMetricsExportInterval());
    boltMetrics.registerMetrics(topologyContext);
    // Initialize the instanceState if the topology is stateful and bolt is a stateful component
    if (isTopologyStateful && bolt instanceof IStatefulComponent) {
        this.instanceState = state;
        ((IStatefulComponent<Serializable, Serializable>) bolt).initState(instanceState);
        if (spillState) {
            if (FileUtils.isDirectoryExists(spillStateLocation)) {
                FileUtils.cleanDir(spillStateLocation);
            } else {
                FileUtils.createDirectory(spillStateLocation);
            }
        }
    }
    // Delegate
    bolt.prepare(topologyContext.getTopologyConfig(), topologyContext, new OutputCollector(collector));
    // Invoke user-defined prepare task hook
    topologyContext.invokeHookPrepare();
    // Init the CustomStreamGrouping
    helper.prepareForCustomStreamGrouping();
}