Examples with MockedSources org.apache.storm.testing.MockedSources used on opensource projects

Example 6 with MockedSources

use of org.apache.storm.testing.MockedSources in project storm by apache.

the class TopologyIntegrationTest method testSystemStream.

@Test
public void testSystemStream() throws Exception {
    // this test works because mocking a spout splits up the tuples evenly among the tasks
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 3);
        builder.setBolt("2", new IdentityBolt(), 1).fieldsGrouping("1", new Fields("word")).globalGrouping("1", "__system");
        StormTopology topology = builder.createTopology();
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_WORKERS, 2);
        List<FixedTuple> testTuples = Arrays.asList("a", "b", "c").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("a"), new Values("b"), new Values("c")));
    }
}

Example 7 with MockedSources

use of org.apache.storm.testing.MockedSources in project storm by apache.

the class MessagingTest method testLocalTransport.

@Test
public void testLocalTransport() throws Exception {
    Config topoConf = new Config();
    topoConf.put(Config.TOPOLOGY_WORKERS, 2);
    topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(1).withPortsPerSupervisor(2).withDaemonConf(topoConf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 2);
        builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1");
        StormTopology stormTopology = builder.createTopology();
        List<FixedTuple> fixedTuples = new ArrayList<>();
        for (int i = 0; i < 12; i++) {
            fixedTuples.add(new FixedTuple(Collections.singletonList("a")));
            fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
        }
        Map<String, List<FixedTuple>> data = new HashMap<>();
        data.put("1", fixedTuples);
        MockedSources mockedSources = new MockedSources(data);
        CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
        completeTopologyParam.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}

Example 8 with MockedSources

use of org.apache.storm.testing.MockedSources in project open-kilda by telstra.

the class OpenTsdbTopologyTest method shouldSuccessfulSendDatapoint.

@Test
public void shouldSuccessfulSendDatapoint() {
    Datapoint datapoint = new Datapoint("metric", timestamp, Collections.emptyMap(), 123);
    MockedSources sources = new MockedSources();
    Testing.withTrackedCluster(clusterParam, (cluster) -> {
        OpenTsdbTopology topology = new OpenTsdbTopology(makeLaunchEnvironment(getProperties()));
        sources.addMockData(ZooKeeperSpout.SPOUT_ID, new Values(LifecycleEvent.builder().signal(Signal.NONE).build(), null));
        sources.addMockData(OpenTsdbTopology.OTSDB_SPOUT_ID, new Values(null, datapoint));
        completeTopologyParam.setMockedSources(sources);
        StormTopology stormTopology = topology.createTopology();
        stormTopology.get_bolts().remove(ZooKeeperBolt.BOLT_ID);
        activateDatapointParserBolt(stormTopology);
        Map result = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
    });
    // verify that request is sent to OpenTSDB server
    mockServer.verify(REQUEST, VerificationTimes.exactly(1));
}

Example 9 with MockedSources

use of org.apache.storm.testing.MockedSources in project ignite by apache.

the class StormIgniteStreamerSelfTest method testStormStreamerIgniteBolt.

/**
 * Tests for the streamer bolt. Ignite started in bolt based on what is specified in the configuration file.
 *
 * @throws TimeoutException
 * @throws InterruptedException
 */
public void testStormStreamerIgniteBolt() throws TimeoutException, InterruptedException {
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();
    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(TEST_CACHE);
    stormStreamer.setIgniteTupleField(TestStormSpout.IGNITE_TUPLE_FIELD);
    stormStreamer.setIgniteConfigFile(GRID_CONF_FILE);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setDaemonConf(daemonConf);
    mkClusterParam.setSupervisors(4);
    final CountDownLatch latch = new CountDownLatch(TestStormSpout.CNT);
    IgniteBiPredicate<UUID, CacheEvent> putLsnr = new IgniteBiPredicate<UUID, CacheEvent>() {

        @Override
        public boolean apply(UUID uuid, CacheEvent evt) {
            assert evt != null;
            latch.countDown();
            return true;
        }
    };
    final UUID putLsnrId = ignite.events(ignite.cluster().forCacheNodes(TEST_CACHE)).remoteListen(putLsnr, null, EVT_CACHE_OBJECT_PUT);
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) throws IOException, InterruptedException {
            // Creates a test topology.
            TopologyBuilder builder = new TopologyBuilder();
            TestStormSpout testStormSpout = new TestStormSpout();
            builder.setSpout("test-spout", testStormSpout);
            builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("test-spout");
            StormTopology topology = builder.createTopology();
            // Prepares a mock data for the spout.
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("test-spout", getMockData());
            // Prepares the config.
            Config conf = new Config();
            conf.setMessageTimeoutSecs(10);
            IgniteCache<Integer, String> cache = ignite.cache(TEST_CACHE);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setTimeoutMs(10000);
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            // Checks the cache doesn't contain any entries yet.
            assertEquals(0, cache.size(CachePeekMode.PRIMARY));
            Testing.completeTopology(cluster, topology, completeTopologyParam);
            // Checks events successfully processed in 20 seconds.
            assertTrue(latch.await(10, TimeUnit.SECONDS));
            ignite.events(ignite.cluster().forCacheNodes(TEST_CACHE)).stopRemoteListen(putLsnrId);
            // Validates all entries are in the cache.
            assertEquals(TestStormSpout.CNT, cache.size(CachePeekMode.PRIMARY));
            for (Map.Entry<Integer, String> entry : TestStormSpout.getKeyValMap().entrySet()) assertEquals(entry.getValue(), cache.get(entry.getKey()));
        }
    });
}

Example 10 with MockedSources

use of org.apache.storm.testing.MockedSources in project open-kilda by telstra.

the class OpenTSDBTopologyTest method shouldSendDatapointRequestsOnlyOnce.

@Ignore
@Test
public void shouldSendDatapointRequestsOnlyOnce() throws Exception {
    Datapoint datapoint = new Datapoint("metric", timestamp, Collections.emptyMap(), 123);
    String jsonDatapoint = MAPPER.writeValueAsString(datapoint);
    MockedSources sources = new MockedSources();
    sources.addMockData(Topic.OTSDB + "-spout", new Values(jsonDatapoint), new Values(jsonDatapoint));
    completeTopologyParam.setMockedSources(sources);
    Testing.withTrackedCluster(clusterParam, (cluster) -> {
        OpenTSDBTopology topology = new TestingTargetTopology(new TestingKafkaBolt());
        StormTopology stormTopology = topology.createTopology();
        Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
    });
    // verify that request is sent to OpenTSDB server once
    mockServer.verify(HttpRequest.request(), VerificationTimes.exactly(1));
}