Examples with CompleteTopologyParam org.apache.storm.testing.CompleteTopologyParam used on opensource projects

Example 6 with CompleteTopologyParam

use of org.apache.storm.testing.CompleteTopologyParam in project storm by apache.

the class TopologyIntegrationTest method testSystemStream.

@Test
public void testSystemStream() throws Exception {
    // this test works because mocking a spout splits up the tuples evenly among the tasks
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 3);
        builder.setBolt("2", new IdentityBolt(), 1).fieldsGrouping("1", new Fields("word")).globalGrouping("1", "__system");
        StormTopology topology = builder.createTopology();
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_WORKERS, 2);
        List<FixedTuple> testTuples = Arrays.asList("a", "b", "c").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("a"), new Values("b"), new Values("c")));
    }
}

Example 7 with CompleteTopologyParam

use of org.apache.storm.testing.CompleteTopologyParam in project storm by apache.

the class MessagingTest method testLocalTransport.

@Test
public void testLocalTransport() throws Exception {
    Config topoConf = new Config();
    topoConf.put(Config.TOPOLOGY_WORKERS, 2);
    topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(1).withPortsPerSupervisor(2).withDaemonConf(topoConf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 2);
        builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1");
        StormTopology stormTopology = builder.createTopology();
        List<FixedTuple> fixedTuples = new ArrayList<>();
        for (int i = 0; i < 12; i++) {
            fixedTuples.add(new FixedTuple(Collections.singletonList("a")));
            fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
        }
        Map<String, List<FixedTuple>> data = new HashMap<>();
        data.put("1", fixedTuples);
        MockedSources mockedSources = new MockedSources(data);
        CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
        completeTopologyParam.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}

Example 8 with CompleteTopologyParam

use of org.apache.storm.testing.CompleteTopologyParam in project open-kilda by telstra.

the class StableAbstractStormTest method startCompleteTopology.

protected static void startCompleteTopology() throws Exception {
    log.info("Starting CompleteTopology ...");
    clusterParam = new MkClusterParam();
    clusterParam.setSupervisors(1);
    Config daemonConfig = new Config();
    daemonConfig.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    clusterParam.setDaemonConf(daemonConfig);
    makeConfigFile();
    Config conf = new Config();
    conf.setNumWorkers(1);
    completeTopologyParam = new CompleteTopologyParam();
    completeTopologyParam.setStormConf(conf);
}

Example 9 with CompleteTopologyParam

use of org.apache.storm.testing.CompleteTopologyParam in project ignite by apache.

the class StormIgniteStreamerSelfTest method testStormStreamerIgniteBolt.

/**
 * Tests for the streamer bolt. Ignite started in bolt based on what is specified in the configuration file.
 *
 * @throws TimeoutException
 * @throws InterruptedException
 */
public void testStormStreamerIgniteBolt() throws TimeoutException, InterruptedException {
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();
    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(TEST_CACHE);
    stormStreamer.setIgniteTupleField(TestStormSpout.IGNITE_TUPLE_FIELD);
    stormStreamer.setIgniteConfigFile(GRID_CONF_FILE);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setDaemonConf(daemonConf);
    mkClusterParam.setSupervisors(4);
    final CountDownLatch latch = new CountDownLatch(TestStormSpout.CNT);
    IgniteBiPredicate<UUID, CacheEvent> putLsnr = new IgniteBiPredicate<UUID, CacheEvent>() {

        @Override
        public boolean apply(UUID uuid, CacheEvent evt) {
            assert evt != null;
            latch.countDown();
            return true;
        }
    };
    final UUID putLsnrId = ignite.events(ignite.cluster().forCacheNodes(TEST_CACHE)).remoteListen(putLsnr, null, EVT_CACHE_OBJECT_PUT);
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) throws IOException, InterruptedException {
            // Creates a test topology.
            TopologyBuilder builder = new TopologyBuilder();
            TestStormSpout testStormSpout = new TestStormSpout();
            builder.setSpout("test-spout", testStormSpout);
            builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("test-spout");
            StormTopology topology = builder.createTopology();
            // Prepares a mock data for the spout.
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("test-spout", getMockData());
            // Prepares the config.
            Config conf = new Config();
            conf.setMessageTimeoutSecs(10);
            IgniteCache<Integer, String> cache = ignite.cache(TEST_CACHE);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setTimeoutMs(10000);
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            // Checks the cache doesn't contain any entries yet.
            assertEquals(0, cache.size(CachePeekMode.PRIMARY));
            Testing.completeTopology(cluster, topology, completeTopologyParam);
            // Checks events successfully processed in 20 seconds.
            assertTrue(latch.await(10, TimeUnit.SECONDS));
            ignite.events(ignite.cluster().forCacheNodes(TEST_CACHE)).stopRemoteListen(putLsnrId);
            // Validates all entries are in the cache.
            assertEquals(TestStormSpout.CNT, cache.size(CachePeekMode.PRIMARY));
            for (Map.Entry<Integer, String> entry : TestStormSpout.getKeyValMap().entrySet()) assertEquals(entry.getValue(), cache.get(entry.getKey()));
        }
    });
}

Example 10 with CompleteTopologyParam

use of org.apache.storm.testing.CompleteTopologyParam in project storm by apache.

the class MessagingTest method testRemoteTransportWithManyTasksInReceivingExecutor.

@Test
public void testRemoteTransportWithManyTasksInReceivingExecutor() throws Exception {
    // STORM-3141 regression test
    // Verify that remote worker can handle many tasks in one executor
    Config topoConf = new Config();
    topoConf.put(Config.TOPOLOGY_WORKERS, 2);
    topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(1).withPortsPerSupervisor(2).withDaemonConf(topoConf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 1);
        builder.setBolt("2", new TestGlobalCount(), 1).setNumTasks(10).shuffleGrouping("1");
        StormTopology stormTopology = builder.createTopology();
        List<FixedTuple> fixedTuples = new ArrayList<>();
        for (int i = 0; i < 12; i++) {
            fixedTuples.add(new FixedTuple(Collections.singletonList("a")));
            fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
        }
        Map<String, List<FixedTuple>> data = new HashMap<>();
        data.put("1", fixedTuples);
        MockedSources mockedSources = new MockedSources(data);
        CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
        completeTopologyParam.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}