Examples with PhysicalPlanHelper com.twitter.heron.common.utils.misc.PhysicalPlanHelper used on opensource projects

Example 1 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class SpoutInstanceTest method testDoImmediateAcks.

/**
   * Test with the acking immediately
   */
@Test
public void testDoImmediateAcks() throws Exception {
    physicalPlan = UnitTestHelper.getPhysicalPlan(false, -1);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    SingletonRegistry.INSTANCE.registerSingleton(Constants.ACK_COUNT, ackCount);
    inControlQueue.offer(instanceControlMsg);
    Runnable task = new Runnable() {

        @Override
        public void run() {
            while (outStreamQueue.size() != 0) {
                HeronTuples.HeronTupleSet set = outStreamQueue.poll();
                Assert.assertTrue(set.isInitialized());
                Assert.assertTrue(set.hasData());
                HeronTuples.HeronDataTupleSet dataTupleSet = set.getData();
                tupleReceived += dataTupleSet.getTuplesCount();
                heronDataTupleList.addAll(dataTupleSet.getTuplesList());
            }
            if (tupleReceived == 10) {
                // We fetch it from SingletonRegistry
                for (int i = 0; i < Constants.RETRY_TIMES; i++) {
                    if (ackCount.intValue() != 0) {
                        break;
                    }
                    SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
                }
                // Wait the bolt's finishing
                SysUtils.sleep(Constants.TEST_WAIT_TIME_MS);
                Assert.assertEquals(10, ackCount.intValue());
                testLooper.exitLoop();
            }
        }
    };
    testLooper.addTasksOnWakeup(task);
    testLooper.loop();
}

Example 2 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class SpoutInstanceTest method testAckAndFail.

/**
   * We will receive tuples and then send back the corresponding ack&fail tuples
   */
@Test
public void testAckAndFail() throws Exception {
    physicalPlan = UnitTestHelper.getPhysicalPlan(true, -1);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    SingletonRegistry.INSTANCE.registerSingleton(Constants.ACK_COUNT, ackCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.FAIL_COUNT, failCount);
    inControlQueue.offer(instanceControlMsg);
    Runnable task = new Runnable() {

        @Override
        public void run() {
            while (outStreamQueue.size() != 0) {
                HeronTuples.HeronTupleSet set = outStreamQueue.poll();
                Assert.assertTrue(set.isInitialized());
                Assert.assertTrue(set.hasData());
                HeronTuples.HeronDataTupleSet dataTupleSet = set.getData();
                tupleReceived += dataTupleSet.getTuplesCount();
                heronDataTupleList.addAll(dataTupleSet.getTuplesList());
            }
            if (tupleReceived == 10) {
                constructAndSendAcks();
                // We fetch it from SingletonRegistry
                for (int i = 0; i < Constants.RETRY_TIMES; i++) {
                    if (ackCount.intValue() != 0 || failCount.intValue() != 0) {
                        break;
                    }
                    SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
                }
                // Wait the bolt's finishing
                SysUtils.sleep(Constants.TEST_WAIT_TIME_MS);
                Assert.assertEquals(5, ackCount.intValue());
                Assert.assertEquals(5, failCount.intValue());
                testLooper.exitLoop();
            }
        }
    };
    testLooper.addTasksOnWakeup(task);
    testLooper.loop();
}

Example 3 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class ConnectTest method testStart.

/**
   * Test connection
   */
@Test
public void testStart() throws Exception {
    ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
    serverSocketChannel.socket().bind(new InetSocketAddress(HOST, serverPort));
    SocketChannel socketChannel = null;
    try {
        runStreamManagerClient();
        socketChannel = serverSocketChannel.accept();
        configure(socketChannel);
        socketChannel.configureBlocking(false);
        close(serverSocketChannel);
        // Receive request
        IncomingPacket incomingPacket = new IncomingPacket();
        while (incomingPacket.readFromChannel(socketChannel) != 0) {
            // 1ms sleep to mitigate busy looping
            SysUtils.sleep(1);
        }
        // Send back response
        // Though we do not use typeName, we need to unpack it first,
        // since the order is required
        String typeName = incomingPacket.unpackString();
        REQID rid = incomingPacket.unpackREQID();
        OutgoingPacket outgoingPacket = new OutgoingPacket(rid, UnitTestHelper.getRegisterInstanceResponse());
        outgoingPacket.writeToChannel(socketChannel);
        for (int i = 0; i < Constants.RETRY_TIMES; i++) {
            InstanceControlMsg instanceControlMsg = inControlQueue.poll();
            if (instanceControlMsg != null) {
                nioLooper.exitLoop();
                threadsPool.shutdownNow();
                PhysicalPlanHelper physicalPlanHelper = instanceControlMsg.getNewPhysicalPlanHelper();
                Assert.assertEquals("test-bolt", physicalPlanHelper.getMyComponent());
                Assert.assertEquals(InetAddress.getLocalHost().getHostName(), physicalPlanHelper.getMyHostname());
                Assert.assertEquals(0, physicalPlanHelper.getMyInstanceIndex());
                Assert.assertEquals(1, physicalPlanHelper.getMyTaskId());
                break;
            } else {
                SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
            }
        }
    } catch (ClosedChannelException ignored) {
    } finally {
        close(socketChannel);
    }
}

Example 4 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class Slave method handleControlMessage.

private void handleControlMessage() {
    Runnable handleControlMessageTask = new Runnable() {

        @Override
        public void run() {
            while (!inControlQueue.isEmpty()) {
                InstanceControlMsg instanceControlMsg = inControlQueue.poll();
                // Handle New Physical Plan
                if (instanceControlMsg.isNewPhysicalPlanHelper()) {
                    PhysicalPlanHelper newHelper = instanceControlMsg.getNewPhysicalPlanHelper();
                    // Bind the MetricsCollector with topologyContext
                    newHelper.setTopologyContext(metricsCollector);
                    if (helper == null) {
                        handleNewAssignment(newHelper);
                    } else {
                        instance.update(newHelper);
                        // Handle the state changing
                        if (!helper.getTopologyState().equals(newHelper.getTopologyState())) {
                            switch(newHelper.getTopologyState()) {
                                case RUNNING:
                                    if (!isInstanceStarted) {
                                        // Start the instance if it has not yet started
                                        startInstance();
                                    }
                                    instance.activate();
                                    break;
                                case PAUSED:
                                    instance.deactivate();
                                    break;
                                default:
                                    throw new RuntimeException("Unexpected TopologyState is updated for spout: " + newHelper.getTopologyState());
                            }
                        } else {
                            LOG.info("Topology state remains the same in Slave: " + helper.getTopologyState());
                        }
                    }
                    // update the PhysicalPlanHelper in Slave
                    helper = newHelper;
                }
            // TODO:- We might handle more control Message in future
            }
        }
    };
    slaveLooper.addTasksOnWakeup(handleControlMessageTask);
}

Example 5 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class StreamManagerClient method handleAssignmentMessage.

private void handleAssignmentMessage(PhysicalPlans.PhysicalPlan pplan) {
    LOG.fine("Physical Plan: " + pplan);
    PhysicalPlanHelper newHelper = new PhysicalPlanHelper(pplan, instance.getInstanceId());
    if (helper != null && (!helper.getMyComponent().equals(newHelper.getMyComponent()) || helper.getMyTaskId() != newHelper.getMyTaskId())) {
        // we will get the new assignment
        throw new RuntimeException("Our Assignment has changed. We will die to pick it");
    }
    if (helper == null) {
        LOG.info("We received a new Physical Plan.");
    } else {
        LOG.info("We received a new Physical Plan with same assignment. Should be state changes.");
        LOG.info(String.format("Old state: %s; new sate: %s.", helper.getTopologyState(), newHelper.getTopologyState()));
    }
    helper = newHelper;
    LOG.info("Push to Slave");
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(helper).build();
    inControlQueue.offer(instanceControlMsg);
}