Examples with FixedTuple org.apache.storm.testing.FixedTuple used on opensource projects

Example 1 with FixedTuple

use of org.apache.storm.testing.FixedTuple in project open-kilda by telstra.

the class StatsTopologyTest method meterConfigStatsTest.

@Test
public void meterConfigStatsTest() throws Exception {
    final String switchId = "00:00:00:00:00:00:00:01";
    final List<MeterConfigReply> stats = Collections.singletonList(new MeterConfigReply(2, Arrays.asList(1L, 2L, 3L)));
    InfoMessage message = new InfoMessage(new MeterConfigStatsData(switchId, stats), timestamp, CORRELATION_ID, Destination.WFM_STATS);
    // mock kafka spout
    MockedSources sources = new MockedSources();
    sources.addMockData(StatsComponentType.STATS_OFS_KAFKA_SPOUT.toString(), new Values(MAPPER.writeValueAsString(message)));
    completeTopologyParam.setMockedSources(sources);
    // execute topology
    Testing.withTrackedCluster(clusterParam, (cluster) -> {
        StatsTopology topology = new TestingTargetTopology(new TestingKafkaBolt());
        StormTopology stormTopology = topology.createTopology();
        // verify results
        Map result = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        ArrayList<FixedTuple> tuples = (ArrayList<FixedTuple>) result.get(StatsComponentType.METER_CFG_STATS_METRIC_GEN.name());
        assertThat(tuples.size(), is(3));
        tuples.stream().map(this::readFromJson).forEach(datapoint -> {
            assertThat(datapoint.getTags().get("switchid"), is(switchId.replaceAll(":", "")));
            assertThat(datapoint.getTime(), is(timestamp));
            assertThat(datapoint.getMetric(), is("pen.switch.meters"));
        });
    });
}

Example 2 with FixedTuple

use of org.apache.storm.testing.FixedTuple in project storm by apache.

the class TopologyIntegrationTest method testComponentSpecificConfig.

@Test
public void testComponentSpecificConfig() throws Exception {
    Map<String, Object> daemonConf = new HashMap<>();
    daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(daemonConf).build()) {
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
        Map<String, Object> componentConf = new HashMap<>();
        componentConf.put("fake.config", 123);
        componentConf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 20);
        componentConf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30);
        componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections.singletonMap("fake.type", "bad.serializer"), Collections.singletonMap("fake.type2", "a.serializer")));
        topologyBuilder.setBolt("2", new TestConfBolt(componentConf)).shuffleGrouping("1").setMaxTaskParallelism(2).addConfiguration("fake.config2", 987);
        List<FixedTuple> testTuples = Arrays.asList("fake.config", Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER).stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        Map<String, String> kryoRegister = new HashMap<>();
        kryoRegister.put("fake.type", "good.serializer");
        kryoRegister.put("fake.type3", "a.serializer3");
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(kryoRegister));
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams);
        Map<String, Object> expectedValues = new HashMap<>();
        expectedValues.put("fake.config", 123L);
        expectedValues.put("fake.config2", 987L);
        expectedValues.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 2L);
        expectedValues.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30L);
        Map<String, String> expectedKryoRegister = new HashMap<>();
        expectedKryoRegister.putAll(kryoRegister);
        expectedKryoRegister.put("fake.type2", "a.serializer");
        expectedValues.put(Config.TOPOLOGY_KRYO_REGISTER, expectedKryoRegister);
        List<Object> concatValues = Testing.readTuples(results, "2").stream().flatMap(values -> values.stream()).collect(Collectors.toList());
        assertThat(listToMap(concatValues), is(expectedValues));
    }
}

Example 3 with FixedTuple

use of org.apache.storm.testing.FixedTuple in project storm by apache.

the class TopologyIntegrationTest method testBasicTopology.

@ParameterizedTest
@ValueSource(strings = { "true", "false" })
public void testBasicTopology(boolean useLocalMessaging) throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(4).withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 3);
        builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
        builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
        builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
        StormTopology topology = builder.createTopology();
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_WORKERS, 2);
        stormConf.put(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE, true);
        List<FixedTuple> testTuples = Arrays.asList("nathan", "bob", "joey", "nathan").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "1"), containsInAnyOrder(new Values("nathan"), new Values("nathan"), new Values("bob"), new Values("joey")));
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)));
        assertThat(Testing.readTuples(results, "3"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
        assertThat(Testing.readTuples(results, "4"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
    }
}

Example 4 with FixedTuple

use of org.apache.storm.testing.FixedTuple in project storm by apache.

the class Testing method completeTopology.

/**
 * Run a topology to completion capturing all of the messages that are emitted.  This only works when all of the spouts are
 * instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param
 * @param cluster the cluster to submit the topology to
 * @param topology the topology itself
 * @param param parameters to describe how to complete a topology
 * @return a map of the component to the list of tuples it emitted
 * @throws TException on any error from nimbus.
 */
public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology, CompleteTopologyParam param) throws TException, InterruptedException {
    Map<String, List<FixedTuple>> ret = null;
    CapturedTopology<StormTopology> capTopo = captureTopology(topology);
    topology = capTopo.topology;
    String topoName = param.getTopologyName();
    if (topoName == null) {
        topoName = "topologytest-" + Utils.uuid();
    }
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    MockedSources ms = param.getMockedSources();
    if (ms != null) {
        for (Entry<String, List<FixedTuple>> mocked : ms.getData().entrySet()) {
            FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue());
            spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout));
        }
    }
    List<Object> spoutObjects = spouts.values().stream().map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
    for (Object o : spoutObjects) {
        if (!(o instanceof CompletableSpout)) {
            throw new RuntimeException("Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
        }
    }
    for (Object spout : spoutObjects) {
        ((CompletableSpout) spout).startup();
    }
    cluster.submitTopology(topoName, param.getStormConf(), topology);
    if (Time.isSimulating()) {
        cluster.advanceClusterTime(11);
    }
    IStormClusterState state = cluster.getClusterState();
    String topoId = state.getTopoId(topoName).get();
    // Give the topology time to come up without using it to wait for the spouts to complete
    simulateWait(cluster);
    Integer timeoutMs = param.getTimeoutMs();
    if (timeoutMs == null) {
        timeoutMs = TEST_TIMEOUT_MS;
    }
    whileTimeout(timeoutMs, () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout) o).isExhausted()), () -> {
        try {
            simulateWait(cluster);
        } catch (Exception e) {
            throw new RuntimeException();
        }
    });
    KillOptions killOpts = new KillOptions();
    killOpts.set_wait_secs(0);
    cluster.killTopologyWithOpts(topoName, killOpts);
    whileTimeout(timeoutMs, () -> state.assignmentInfo(topoId, null) != null, () -> {
        try {
            simulateWait(cluster);
        } catch (Exception e) {
            throw new RuntimeException();
        }
    });
    if (param.getCleanupState()) {
        for (Object o : spoutObjects) {
            ((CompletableSpout) o).clean();
        }
        ret = capTopo.capturer.getAndRemoveResults();
    } else {
        ret = capTopo.capturer.getAndClearResults();
    }
    return ret;
}

Example 5 with FixedTuple

use of org.apache.storm.testing.FixedTuple in project storm by apache.

the class TopologyIntegrationTest method testSystemStream.

@Test
public void testSystemStream() throws Exception {
    // this test works because mocking a spout splits up the tuples evenly among the tasks
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 3);
        builder.setBolt("2", new IdentityBolt(), 1).fieldsGrouping("1", new Fields("word")).globalGrouping("1", "__system");
        StormTopology topology = builder.createTopology();
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_WORKERS, 2);
        List<FixedTuple> testTuples = Arrays.asList("a", "b", "c").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("a"), new Values("b"), new Values("c")));
    }
}