Examples with PhysicalPlanHelper com.twitter.heron.common.utils.misc.PhysicalPlanHelper used on opensource projects

Example 11 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project incubator-heron by apache.

the class ActivateDeactivateTest method buildMessage.

private InstanceControlMsg buildMessage(TopologyAPI.TopologyState state) {
    PhysicalPlans.PhysicalPlan physicalPlan = UnitTestHelper.getPhysicalPlan(true, -1, state);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    return InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
}

Example 12 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class CustomGroupingTest method testCustomGrouping.

/**
   * Test custom grouping
   */
@Test
public void testCustomGrouping() throws Exception {
    final MyCustomGrouping myCustomGrouping = new MyCustomGrouping();
    final String expectedCustomGroupingStringInPrepare = "test-spout+test-spout+default+[1]";
    physicalPlan = constructPhysicalPlan(myCustomGrouping);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    SingletonRegistry.INSTANCE.registerSingleton(CUSTOM_GROUPING_INFO, customGroupingInfoInPrepare);
    Runnable task = new Runnable() {

        @Override
        public void run() {
            for (int i = 0; i < Constants.RETRY_TIMES; i++) {
                if (outStreamQueue.size() != 0) {
                    HeronTuples.HeronTupleSet set = outStreamQueue.poll();
                    Assert.assertTrue(set.isInitialized());
                    Assert.assertFalse(set.hasControl());
                    Assert.assertTrue(set.hasData());
                    HeronTuples.HeronDataTupleSet dataTupleSet = set.getData();
                    Assert.assertEquals(dataTupleSet.getStream().getId(), "default");
                    Assert.assertEquals(dataTupleSet.getStream().getComponentName(), "test-spout");
                    for (HeronTuples.HeronDataTuple dataTuple : dataTupleSet.getTuplesList()) {
                        List<Integer> destTaskIds = dataTuple.getDestTaskIdsList();
                        Assert.assertEquals(destTaskIds.size(), 1);
                        Assert.assertEquals(destTaskIds.get(0), (Integer) tupleReceived);
                        tupleReceived++;
                    }
                }
                if (tupleReceived == 10) {
                    Assert.assertEquals(expectedCustomGroupingStringInPrepare, customGroupingInfoInPrepare.toString());
                    testLooper.exitLoop();
                    break;
                }
                SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
            }
        }
    };
    testLooper.addTasksOnWakeup(task);
    testLooper.loop();
}

Example 13 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class BoltInstanceTest method testReadTupleAndExecute.

/**
   * Test the reading of a tuple and apply execute on that tuple
   */
@Test
public void testReadTupleAndExecute() throws Exception {
    physicalPlan = UnitTestHelper.getPhysicalPlan(false, -1);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, BOLT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.ACK_COUNT, ackCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.FAIL_COUNT, failCount);
    SingletonRegistry.INSTANCE.registerSingleton("execute-count", tupleExecutedCount);
    SingletonRegistry.INSTANCE.registerSingleton("received-string-list", receivedStrings);
    // Send tuples to bolt instance
    HeronTuples.HeronTupleSet.Builder heronTupleSet = HeronTuples.HeronTupleSet.newBuilder();
    HeronTuples.HeronDataTupleSet.Builder dataTupleSet = HeronTuples.HeronDataTupleSet.newBuilder();
    TopologyAPI.StreamId.Builder streamId = TopologyAPI.StreamId.newBuilder();
    streamId.setComponentName("test-spout");
    streamId.setId("default");
    dataTupleSet.setStream(streamId);
    // We will add 10 tuples to the set
    for (int i = 0; i < 10; i++) {
        HeronTuples.HeronDataTuple.Builder dataTuple = HeronTuples.HeronDataTuple.newBuilder();
        dataTuple.setKey(19901017 + i);
        HeronTuples.RootId.Builder rootId = HeronTuples.RootId.newBuilder();
        rootId.setKey(19901017 + i);
        rootId.setTaskid(0);
        dataTuple.addRoots(rootId);
        String s = "";
        if ((i & 1) == 0) {
            s = "A";
        } else {
            s = "B";
        }
        ByteString byteString = ByteString.copyFrom(serializer.serialize(s));
        dataTuple.addValues(byteString);
        dataTupleSet.addTuples(dataTuple);
    }
    heronTupleSet.setData(dataTupleSet);
    inStreamQueue.offer(heronTupleSet.build());
    for (int i = 0; i < Constants.RETRY_TIMES; i++) {
        if (tupleExecutedCount.intValue() == 10) {
            break;
        }
        SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
    }
    // Wait the bolt's finishing
    SysUtils.sleep(Constants.TEST_WAIT_TIME_MS);
    Assert.assertEquals(10, tupleExecutedCount.intValue());
    Assert.assertEquals(5, ackCount.intValue());
    Assert.assertEquals(5, failCount.intValue());
    Assert.assertEquals("ABABABABAB", receivedStrings.toString());
}

Example 14 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class ActivateDeactivateTest method testActivateAndDeactivate.

/**
   * We will test whether spout would pull activate/deactivate state change and
   * invoke activate()/deactivate()
   */
@Test
public void testActivateAndDeactivate() throws Exception {
    physicalPlan = UnitTestHelper.getPhysicalPlan(true, -1, TopologyAPI.TopologyState.RUNNING);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    AtomicInteger activateCount = new AtomicInteger(0);
    AtomicInteger deactivateCount = new AtomicInteger(0);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.ACTIVATE_COUNT, activateCount);
    SingletonRegistry.INSTANCE.registerSingleton(Constants.DEACTIVATE_COUNT, deactivateCount);
    // Now the activateCount and deactivateCount should be 0
    // And we start the test
    physicalPlan = UnitTestHelper.getPhysicalPlan(true, -1, TopologyAPI.TopologyState.PAUSED);
    physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    Thread.sleep(Constants.TEST_WAIT_TIME_MS);
    Assert.assertEquals(1, deactivateCount.get());
    physicalPlan = UnitTestHelper.getPhysicalPlan(true, -1, TopologyAPI.TopologyState.RUNNING);
    physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    Thread.sleep(Constants.TEST_WAIT_TIME_MS);
    Assert.assertEquals(1, activateCount.get());
    Assert.assertEquals(1, deactivateCount.get());
}

Example 15 with PhysicalPlanHelper

use of com.twitter.heron.common.utils.misc.PhysicalPlanHelper in project heron by twitter.

the class SpoutInstanceTest method testNextTuple.

/**
   * Test the fetching of next tuple
   */
@Test
public void testNextTuple() throws Exception {
    physicalPlan = UnitTestHelper.getPhysicalPlan(false, -1);
    PhysicalPlanHelper physicalPlanHelper = new PhysicalPlanHelper(physicalPlan, SPOUT_INSTANCE_ID);
    InstanceControlMsg instanceControlMsg = InstanceControlMsg.newBuilder().setNewPhysicalPlanHelper(physicalPlanHelper).build();
    inControlQueue.offer(instanceControlMsg);
    Runnable task = new Runnable() {

        private String streamId = "";

        private String componentName = "";

        private String receivedTupleStrings = "";

        @Override
        public void run() {
            for (int i = 0; i < Constants.RETRY_TIMES; i++) {
                if (outStreamQueue.size() != 0) {
                    HeronTuples.HeronTupleSet set = outStreamQueue.poll();
                    Assert.assertTrue(set.isInitialized());
                    Assert.assertFalse(set.hasControl());
                    Assert.assertTrue(set.hasData());
                    HeronTuples.HeronDataTupleSet dataTupleSet = set.getData();
                    streamId = dataTupleSet.getStream().getId();
                    componentName = dataTupleSet.getStream().getComponentName();
                    Assert.assertEquals(streamId, "default");
                    Assert.assertEquals(componentName, "test-spout");
                    Assert.assertEquals(streamId, "default");
                    Assert.assertEquals(componentName, "test-spout");
                    for (HeronTuples.HeronDataTuple dataTuple : dataTupleSet.getTuplesList()) {
                        for (ByteString b : dataTuple.getValuesList()) {
                            receivedTupleStrings += serializer.deserialize(b.toByteArray());
                        }
                    }
                    tupleReceived += dataTupleSet.getTuplesCount();
                }
                if (tupleReceived == 10) {
                    Assert.assertEquals("ABABABABAB", receivedTupleStrings);
                    testLooper.exitLoop();
                    break;
                }
                SysUtils.sleep(Constants.RETRY_INTERVAL_MS);
            }
        }
    };
    testLooper.addTasksOnWakeup(task);
    testLooper.loop();
    Assert.assertEquals(tupleReceived, 10);
}