Search in sources :

Example 1 with CompletableSpout

use of org.apache.storm.testing.CompletableSpout in project storm by apache.

the class Testing method completeTopology.

/**
     * Run a topology to completion capturing all of the messages that are emitted.  This only works when all of the spouts are
     * instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param
     * @param cluster the cluster to submit the topology to
     * @param topology the topology itself
     * @param param parameters to describe how to complete a topology.
     * @return a map of the component to the list of tuples it emitted. 
     * @throws InterruptedException
     * @throws TException on any error from nimbus.
     */
public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology, CompleteTopologyParam param) throws TException, InterruptedException {
    Map<String, List<FixedTuple>> ret = null;
    IStormClusterState state = cluster.getClusterState();
    CapturedTopology<StormTopology> capTopo = captureTopology(topology);
    topology = capTopo.topology;
    String topoName = param.getTopologyName();
    if (topoName == null) {
        topoName = "topologytest-" + Utils.uuid();
    }
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    MockedSources ms = param.getMockedSources();
    if (ms != null) {
        for (Entry<String, List<FixedTuple>> mocked : ms.getData().entrySet()) {
            FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue());
            spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout));
        }
    }
    List<Object> spoutObjects = spouts.values().stream().map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
    for (Object o : spoutObjects) {
        if (!(o instanceof CompletableSpout)) {
            throw new RuntimeException("Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
        }
    }
    for (Object spout : spoutObjects) {
        ((CompletableSpout) spout).startup();
    }
    cluster.submitTopology(topoName, param.getStormConf(), topology);
    if (Time.isSimulating()) {
        cluster.advanceClusterTime(11);
    }
    String topoId = state.getTopoId(topoName).get();
    //Give the topology time to come up without using it to wait for the spouts to complete
    simulateWait(cluster);
    Integer timeoutMs = param.getTimeoutMs();
    if (timeoutMs == null) {
        timeoutMs = TEST_TIMEOUT_MS;
    }
    whileTimeout(timeoutMs, () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout) o).isExhausted()), () -> {
        try {
            simulateWait(cluster);
        } catch (Exception e) {
            throw new RuntimeException();
        }
    });
    KillOptions killOpts = new KillOptions();
    killOpts.set_wait_secs(0);
    cluster.killTopologyWithOpts(topoName, killOpts);
    whileTimeout(timeoutMs, () -> state.assignmentInfo(topoId, null) != null, () -> {
        try {
            simulateWait(cluster);
        } catch (Exception e) {
            throw new RuntimeException();
        }
    });
    if (param.getCleanupState()) {
        for (Object o : spoutObjects) {
            ((CompletableSpout) o).clean();
        }
        ret = capTopo.capturer.getAndRemoveResults();
    } else {
        ret = capTopo.capturer.getAndClearResults();
    }
    return ret;
}
Also used : SimulatedTime(org.apache.storm.utils.Time.SimulatedTime) TopologyContext(org.apache.storm.task.TopologyContext) LoggerFactory(org.slf4j.LoggerFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) INimbus(org.apache.storm.scheduler.INimbus) ArrayList(java.util.ArrayList) Bolt(org.apache.storm.generated.Bolt) Tuple(org.apache.storm.tuple.Tuple) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StormTopology(org.apache.storm.generated.StormTopology) Map(java.util.Map) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) MkTupleParam(org.apache.storm.testing.MkTupleParam) FixedTupleSpout(org.apache.storm.testing.FixedTupleSpout) KillOptions(org.apache.storm.generated.KillOptions) MockedSources(org.apache.storm.testing.MockedSources) TestJob(org.apache.storm.testing.TestJob) MkClusterParam(org.apache.storm.testing.MkClusterParam) Logger(org.slf4j.Logger) Predicate(java.util.function.Predicate) Collection(java.util.Collection) CompletableSpout(org.apache.storm.testing.CompletableSpout) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Grouping(org.apache.storm.generated.Grouping) StreamInfo(org.apache.storm.generated.StreamInfo) FixedTuple(org.apache.storm.testing.FixedTuple) TException(org.apache.thrift.TException) Fields(org.apache.storm.tuple.Fields) IStormClusterState(org.apache.storm.cluster.IStormClusterState) Utils(org.apache.storm.utils.Utils) Collectors(java.util.stream.Collectors) GlobalStreamId(org.apache.storm.generated.GlobalStreamId) Time(org.apache.storm.utils.Time) TupleCaptureBolt(org.apache.storm.testing.TupleCaptureBolt) List(java.util.List) RegisteredGlobalState(org.apache.storm.utils.RegisteredGlobalState) ConfigUtils(org.apache.storm.utils.ConfigUtils) TupleImpl(org.apache.storm.tuple.TupleImpl) SpoutSpec(org.apache.storm.generated.SpoutSpec) Entry(java.util.Map.Entry) TrackedTopology(org.apache.storm.testing.TrackedTopology) StormTopology(org.apache.storm.generated.StormTopology) CompletableSpout(org.apache.storm.testing.CompletableSpout) TException(org.apache.thrift.TException) FixedTupleSpout(org.apache.storm.testing.FixedTupleSpout) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MockedSources(org.apache.storm.testing.MockedSources) SpoutSpec(org.apache.storm.generated.SpoutSpec) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) KillOptions(org.apache.storm.generated.KillOptions)

Aggregations

ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 Predicate (java.util.function.Predicate)1 Collectors (java.util.stream.Collectors)1 IStormClusterState (org.apache.storm.cluster.IStormClusterState)1 Bolt (org.apache.storm.generated.Bolt)1 GlobalStreamId (org.apache.storm.generated.GlobalStreamId)1 Grouping (org.apache.storm.generated.Grouping)1 KillOptions (org.apache.storm.generated.KillOptions)1 SpoutSpec (org.apache.storm.generated.SpoutSpec)1 StormTopology (org.apache.storm.generated.StormTopology)1 StreamInfo (org.apache.storm.generated.StreamInfo)1 INimbus (org.apache.storm.scheduler.INimbus)1