Examples with PhysicalPlanHelper org.apache.heron.common.utils.misc.PhysicalPlanHelper used on opensource projects

Example 6 with PhysicalPlanHelper

use of org.apache.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class Executor method handleNewPhysicalPlan.

private void handleNewPhysicalPlan(InstanceControlMsg instanceControlMsg) {
    PhysicalPlanHelper newHelper = instanceControlMsg.getNewPhysicalPlanHelper();
    // Bind the MetricsCollector with topologyContext
    newHelper.setTopologyContext(metricsCollector);
    if (helper == null) {
        helper = newHelper;
        handleNewAssignment();
    } else {
        TopologyAPI.TopologyState oldTopologyState = helper.getTopologyState();
        // Update the PhysicalPlanHelper
        helper = newHelper;
        instance.update(helper);
        // Handle the state changing
        if (!oldTopologyState.equals(helper.getTopologyState())) {
            switch(helper.getTopologyState()) {
                case RUNNING:
                    if (!isInstanceStarted) {
                        // Start the instance if it has not yet started
                        startInstanceIfNeeded();
                    }
                    instance.activate();
                    break;
                case PAUSED:
                    instance.deactivate();
                    break;
                default:
                    throw new RuntimeException("Unexpected TopologyState is updated for spout: " + helper.getTopologyState());
            }
        } else {
            LOG.info("Topology state remains the same in Executor: " + oldTopologyState);
        }
    }
}

Example 7 with PhysicalPlanHelper

use of org.apache.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class AbstractTupleRoutingTest method testRoundRobinRouting.

/**
 * Test that tuple routing occurs using round robin
 */
@Test
public void testRoundRobinRouting() throws Exception {
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(constructPhysicalPlan(), getComponentToVerify().getInstanceId());
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    executorTester.getInControlQueue().offer(instanceControlMsg);
    SingletonRegistry.INSTANCE.registerSingleton(getInitInfoKey(getComponentToVerify().getName()), groupingInitInfo);
    final int expectedTuplesValidated = 10;
    Runnable task = new Runnable() {

        @Override
        public void run() {
            HeronServerTester.await(outStreamQueueOfferLatch);
            assertNotEquals(0, executorTester.getOutStreamQueue().size());
            while (tupleReceived < expectedTuplesValidated) {
                if (executorTester.getOutStreamQueue().isEmpty()) {
                    continue;
                }
                Message msg = executorTester.getOutStreamQueue().poll();
                assertTrue(msg instanceof HeronTuples.HeronTupleSet);
                HeronTuples.HeronTupleSet set = (HeronTuples.HeronTupleSet) msg;
                assertTrue(set.isInitialized());
                assertFalse(set.hasControl());
                assertTrue(set.hasData());
                HeronTuples.HeronDataTupleSet dataTupleSet = set.getData();
                assertEquals(dataTupleSet.getStream().getId(), "default");
                assertEquals(dataTupleSet.getStream().getComponentName(), getComponentToVerify().getName());
                for (HeronTuples.HeronDataTuple dataTuple : dataTupleSet.getTuplesList()) {
                    List<Integer> destTaskIds = dataTuple.getDestTaskIdsList();
                    assertEquals(1, destTaskIds.size());
                    assertEquals((Integer) tupleReceived, destTaskIds.get(0));
                    tupleReceived++;
                }
            }
            assertEquals(expectedTuplesValidated, tupleReceived);
            assertEquals(getExpectedComponentInitInfo(), groupingInitInfo.toString());
            executorTester.getTestLooper().exitLoop();
        }
    };
    executorTester.getTestLooper().addTasksOnWakeup(task);
    executorTester.getTestLooper().loop();
    assertEquals(expectedTuplesValidated, tupleReceived);
}

Example 8 with PhysicalPlanHelper

use of org.apache.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class BoltInstanceTest method testReadTupleAndExecute.

/**
 * Test the reading of a tuple and apply execute on that tuple
 */
@Test
public void testReadTupleAndExecute() {
    PhysicalPlans.PhysicalPlan physicalPlan = UnitTestHelper.getPhysicalPlan(false, -1);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, BOLT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    executorTester.getInControlQueue().offer(instanceControlMsg);
    final int expectedTuples = 10;
    CountDownLatch executeLatch = new CountDownLatch(expectedTuples);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.ACK_COUNT, ackCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.FAIL_COUNT, failCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.EXECUTE_COUNT, tupleExecutedCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.EXECUTE_LATCH, executeLatch);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.RECEIVED_STRING_LIST, receivedStrings);
    // Send tuples to bolt instance
    HeronTuples.HeronTupleSet.Builder heronTupleSet = HeronTuples.HeronTupleSet.newBuilder();
    heronTupleSet.setSrcTaskId(SRC_TASK_ID);
    HeronTuples.HeronDataTupleSet.Builder dataTupleSet = HeronTuples.HeronDataTupleSet.newBuilder();
    TopologyAPI.StreamId.Builder streamId = TopologyAPI.StreamId.newBuilder();
    streamId.setComponentName("test-spout");
    streamId.setId("default");
    dataTupleSet.setStream(streamId);
    // We will add 10 tuples to the set
    for (int i = 0; i < expectedTuples; i++) {
        HeronTuples.HeronDataTuple.Builder dataTuple = HeronTuples.HeronDataTuple.newBuilder();
        dataTuple.setKey(19901017 + i);
        HeronTuples.RootId.Builder rootId = HeronTuples.RootId.newBuilder();
        rootId.setKey(19901017 + i);
        rootId.setTaskid(0);
        dataTuple.addRoots(rootId);
        String tupleValue = (i & 1) == 0 ? "A" : "B";
        dataTuple.addValues(ByteString.copyFrom(serializer.serialize(tupleValue)));
        dataTupleSet.addTuples(dataTuple);
    }
    heronTupleSet.setData(dataTupleSet);
    executorTester.getInStreamQueue().offer(heronTupleSet.build());
    // Wait the bolt's finishing
    HeronServerTester.await(executeLatch);
    Assert.assertEquals(expectedTuples, tupleExecutedCount.intValue());
    Assert.assertEquals(expectedTuples / 2, ackCount.intValue());
    Assert.assertEquals(expectedTuples / 2, failCount.intValue());
    Assert.assertEquals("ABABABABAB", receivedStrings.toString());
}

Example 9 with PhysicalPlanHelper

use of org.apache.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class StreamManagerClient method handleAssignmentMessage.

private void handleAssignmentMessage(PhysicalPlans.PhysicalPlan pplan) {
    LOG.fine("Physical Plan: " + pplan);
    PhysicalPlanHelper newHelper = new PhysicalPlanHelper(pplan, instance.getInstanceId());
    if (helper != null && (!helper.getMyComponent().equals(newHelper.getMyComponent()) || helper.getMyTaskId() != newHelper.getMyTaskId())) {
        // we will get the new assignment
        throw new RuntimeException("Our Assignment has changed. We will die to pick it");
    }
    if (helper == null) {
        LOG.info("We received a new Physical Plan.");
    } else {
        LOG.info("We received a new Physical Plan with same assignment. Should be state changes.");
        LOG.info(String.format("Old state: %s; new sate: %s.", helper.getTopologyState(), newHelper.getTopologyState()));
    }
    helper = newHelper;
    LOG.info("Push to Executor");
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(helper).build();
    inControlQueue.offer(instanceControlMsg);
}

Example 10 with PhysicalPlanHelper

use of org.apache.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class ConnectTest method testStart.

/**
 * Test connection
 */
@Test
public void testStart() throws IOException {
    ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
    serverSocketChannel.socket().bind(new InetSocketAddress(HOST, getServerPort()));
    SocketChannel socketChannel = null;
    try {
        runStreamManagerClient();
        socketChannel = acceptSocketChannel(serverSocketChannel);
        // Receive request
        REQID rid = readIncomingPacket(socketChannel).unpackREQID();
        OutgoingPacket outgoingPacket = new OutgoingPacket(rid, UnitTestHelper.getRegisterInstanceResponse());
        outgoingPacket.writeToChannel(socketChannel);
        HeronServerTester.await(getInControlQueueOfferLatch());
        InstanceControlMsg instanceControlMsg = getInControlQueue().poll();
        assertNotNull(instanceControlMsg);
        getNIOLooper().exitLoop();
        getThreadPool().shutdownNow();
        PhysicalPlanHelper physicalPlanHelper = instanceControlMsg.getNewPhysicalPlanHelper();
        Assert.assertEquals("test-bolt", physicalPlanHelper.getMyComponent());
        Assert.assertEquals(InetAddress.getLocalHost().getHostName(), physicalPlanHelper.getMyHostname());
        Assert.assertEquals(0, physicalPlanHelper.getMyInstanceIndex());
        Assert.assertEquals(1, physicalPlanHelper.getMyTaskId());
    } catch (ClosedChannelException ignored) {
    } finally {
        close(socketChannel);
    }
}