use of org.apache.storm.testing.MockedSources in project open-kilda by telstra.
the class OpenTSDBTopologyTest method shouldSendDatapointRequestsTwice.
@Ignore
@Test
public void shouldSendDatapointRequestsTwice() throws Exception {
Datapoint datapoint1 = new Datapoint("metric", timestamp, Collections.emptyMap(), 123);
String jsonDatapoint1 = MAPPER.writeValueAsString(datapoint1);
Datapoint datapoint2 = new Datapoint("metric", timestamp, Collections.emptyMap(), 456);
String jsonDatapoint2 = MAPPER.writeValueAsString(datapoint2);
MockedSources sources = new MockedSources();
sources.addMockData(Topic.OTSDB + "-spout", new Values(jsonDatapoint1), new Values(jsonDatapoint2));
completeTopologyParam.setMockedSources(sources);
Testing.withTrackedCluster(clusterParam, (cluster) -> {
OpenTSDBTopology topology = new TestingTargetTopology(new TestingKafkaBolt());
StormTopology stormTopology = topology.createTopology();
Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
});
// verify that request is sent to OpenTSDB server once
mockServer.verify(HttpRequest.request(), VerificationTimes.exactly(2));
}
use of org.apache.storm.testing.MockedSources in project open-kilda by telstra.
the class TopologyTest method verifyEmittedValues.
@Test
public void verifyEmittedValues() {
MkClusterParam clusterParam = new MkClusterParam();
clusterParam.setSupervisors(1);
withSimulatedTimeLocalCluster(clusterParam, new TestJob() {
@Override
public void run(ILocalCluster cluster) throws JsonProcessingException {
MockedSources mockedSources = new MockedSources();
mockedSources.addMockData(builder.getSpoutId(), new Values(SWITCH_ID));
Config config = new Config();
config.setDebug(true);
CompleteTopologyParam topologyParam = new CompleteTopologyParam();
topologyParam.setMockedSources(mockedSources);
topologyParam.setStormConf(config);
Map<?, ?> result = completeTopology(cluster, builder.build(), topologyParam);
assertTrue(multiseteq(new Values(new Values(SWITCH_ID)), readTuples(result, builder.getSpoutId())));
assertTrue(multiseteq(new Values(new Values(SWITCH_ID)), readTuples(result, builder.getConfirmationBoltId())));
}
});
}
use of org.apache.storm.testing.MockedSources in project storm by apache.
the class TopologyIntegrationTest method testComponentSpecificConfig.
@Test
public void testComponentSpecificConfig() throws Exception {
Map<String, Object> daemonConf = new HashMap<>();
daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(daemonConf).build()) {
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
Map<String, Object> componentConf = new HashMap<>();
componentConf.put("fake.config", 123);
componentConf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 20);
componentConf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30);
componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections.singletonMap("fake.type", "bad.serializer"), Collections.singletonMap("fake.type2", "a.serializer")));
topologyBuilder.setBolt("2", new TestConfBolt(componentConf)).shuffleGrouping("1").setMaxTaskParallelism(2).addConfiguration("fake.config2", 987);
List<FixedTuple> testTuples = Arrays.asList("fake.config", Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER).stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
Map<String, String> kryoRegister = new HashMap<>();
kryoRegister.put("fake.type", "good.serializer");
kryoRegister.put("fake.type3", "a.serializer3");
Map<String, Object> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(kryoRegister));
MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
completeTopologyParams.setMockedSources(mockedSources);
completeTopologyParams.setStormConf(stormConf);
Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams);
Map<String, Object> expectedValues = new HashMap<>();
expectedValues.put("fake.config", 123L);
expectedValues.put("fake.config2", 987L);
expectedValues.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 2L);
expectedValues.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30L);
Map<String, String> expectedKryoRegister = new HashMap<>();
expectedKryoRegister.putAll(kryoRegister);
expectedKryoRegister.put("fake.type2", "a.serializer");
expectedValues.put(Config.TOPOLOGY_KRYO_REGISTER, expectedKryoRegister);
List<Object> concatValues = Testing.readTuples(results, "2").stream().flatMap(values -> values.stream()).collect(Collectors.toList());
assertThat(listToMap(concatValues), is(expectedValues));
}
}
use of org.apache.storm.testing.MockedSources in project storm by apache.
the class TopologyIntegrationTest method testBasicTopology.
@ParameterizedTest
@ValueSource(strings = { "true", "false" })
public void testBasicTopology(boolean useLocalMessaging) throws Exception {
try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(4).withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)).build()) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("1", new TestWordSpout(true), 3);
builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
StormTopology topology = builder.createTopology();
Map<String, Object> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_WORKERS, 2);
stormConf.put(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE, true);
List<FixedTuple> testTuples = Arrays.asList("nathan", "bob", "joey", "nathan").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
completeTopologyParams.setMockedSources(mockedSources);
completeTopologyParams.setStormConf(stormConf);
Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
assertThat(Testing.readTuples(results, "1"), containsInAnyOrder(new Values("nathan"), new Values("nathan"), new Values("bob"), new Values("joey")));
assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)));
assertThat(Testing.readTuples(results, "3"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
assertThat(Testing.readTuples(results, "4"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
}
}
use of org.apache.storm.testing.MockedSources in project storm by apache.
the class Testing method completeTopology.
/**
* Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are
* instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param
* @param cluster the cluster to submit the topology to
* @param topology the topology itself
* @param param parameters to describe how to complete a topology
* @return a map of the component to the list of tuples it emitted
* @throws TException on any error from nimbus.
*/
public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology, CompleteTopologyParam param) throws TException, InterruptedException {
Map<String, List<FixedTuple>> ret = null;
CapturedTopology<StormTopology> capTopo = captureTopology(topology);
topology = capTopo.topology;
String topoName = param.getTopologyName();
if (topoName == null) {
topoName = "topologytest-" + Utils.uuid();
}
Map<String, SpoutSpec> spouts = topology.get_spouts();
MockedSources ms = param.getMockedSources();
if (ms != null) {
for (Entry<String, List<FixedTuple>> mocked : ms.getData().entrySet()) {
FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue());
spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout));
}
}
List<Object> spoutObjects = spouts.values().stream().map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
for (Object o : spoutObjects) {
if (!(o instanceof CompletableSpout)) {
throw new RuntimeException("Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
}
}
for (Object spout : spoutObjects) {
((CompletableSpout) spout).startup();
}
cluster.submitTopology(topoName, param.getStormConf(), topology);
if (Time.isSimulating()) {
cluster.advanceClusterTime(11);
}
IStormClusterState state = cluster.getClusterState();
String topoId = state.getTopoId(topoName).get();
// Give the topology time to come up without using it to wait for the spouts to complete
simulateWait(cluster);
Integer timeoutMs = param.getTimeoutMs();
if (timeoutMs == null) {
timeoutMs = TEST_TIMEOUT_MS;
}
whileTimeout(timeoutMs, () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout) o).isExhausted()), () -> {
try {
simulateWait(cluster);
} catch (Exception e) {
throw new RuntimeException();
}
});
KillOptions killOpts = new KillOptions();
killOpts.set_wait_secs(0);
cluster.killTopologyWithOpts(topoName, killOpts);
whileTimeout(timeoutMs, () -> state.assignmentInfo(topoId, null) != null, () -> {
try {
simulateWait(cluster);
} catch (Exception e) {
throw new RuntimeException();
}
});
if (param.getCleanupState()) {
for (Object o : spoutObjects) {
((CompletableSpout) o).clean();
}
ret = capTopo.capturer.getAndRemoveResults();
} else {
ret = capTopo.capturer.getAndClearResults();
}
return ret;
}
Aggregations