Examples with SharedMemory org.apache.storm.generated.SharedMemory used on opensource projects

Example 1 with SharedMemory

use of org.apache.storm.generated.SharedMemory in project storm by apache.

the class BatchSubtopologyBuilder method extendTopology.

public void extendTopology(TopologyBuilder builder) {
    BoltDeclarer declarer = builder.setBolt(masterId, new CoordinatedBolt(masterBolt.bolt), masterBolt.parallelism);
    for (InputDeclaration decl : masterBolt.declarations) {
        decl.declare(declarer);
    }
    declarer.addConfigurations(masterBolt.componentConf);
    for (String id : bolts.keySet()) {
        Component component = bolts.get(id);
        Map<String, SourceArgs> coordinatedArgs = new HashMap<String, SourceArgs>();
        for (String c : componentBoltSubscriptions(component)) {
            SourceArgs source;
            if (c.equals(masterId)) {
                source = SourceArgs.single();
            } else {
                source = SourceArgs.all();
            }
            coordinatedArgs.put(c, source);
        }
        BoltDeclarer input = builder.setBolt(id, new CoordinatedBolt(component.bolt, coordinatedArgs, null), component.parallelism);
        for (SharedMemory request : component.sharedMemory) {
            input.addSharedMemory(request);
        }
        if (!component.componentConf.isEmpty()) {
            input.addConfigurations(component.componentConf);
        }
        for (String c : componentBoltSubscriptions(component)) {
            input.directGrouping(c, Constants.COORDINATED_STREAM_ID);
        }
        for (InputDeclaration d : component.declarations) {
            d.declare(input);
        }
    }
}

Example 2 with SharedMemory

use of org.apache.storm.generated.SharedMemory in project storm by apache.

the class Cluster method calculateSharedOffHeapNodeMemory.

private double calculateSharedOffHeapNodeMemory(String nodeId, TopologyDetails td, ExecutorDetails extra) {
    // short-circuit calculation if topology does not use SharedOffHeapMemory
    String topoId = td.getId();
    if (!topoSharedOffHeapMemoryNodeFlag.containsKey(topoId)) {
        initializeTopoSharedOffHeapNodeMemoryFlag(td);
    }
    if (!topoSharedOffHeapMemoryNodeFlag.get(topoId)) {
        return 0.0;
    }
    Set<ExecutorDetails> executorsOnNode = new HashSet<>();
    topoIdToNodeIdToSlotIdToExecutors.computeIfAbsent(td.getId(), Cluster::makeMap).computeIfAbsent(nodeId, Cluster::makeMap).forEach((k, v) -> executorsOnNode.addAll(v));
    if (extra != null) {
        executorsOnNode.add(extra);
    }
    // Now check for overlap on the node
    double memorySharedWithinNode = 0.0;
    for (SharedMemory shared : td.getSharedMemoryRequests(executorsOnNode)) {
        memorySharedWithinNode += shared.get_off_heap_node();
    }
    return memorySharedWithinNode;
}

Example 3 with SharedMemory

use of org.apache.storm.generated.SharedMemory in project storm by apache.

the class LinearDRPCTopologyBuilder method createTopology.

private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < components.size(); i++) {
        Component component = components.get(i);
        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);
        for (SharedMemory request : component.sharedMemory) {
            declarer.addSharedMemory(request);
        }
        if (!component.componentConf.isEmpty()) {
            declarer.addConfigurations(component.componentConf);
        }
        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }
    IRichBolt lastBolt = components.get(components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException("Output stream of last component in LinearDRPCTopology must contain exactly two fields. " + "The first should be the request id, and the second should be the result.");
    }
    builder.setBolt(boltId(i), new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0))).fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt(boltId(i), new ReturnResults()).noneGrouping(boltId(i - 1));
    return builder.createTopology();
}

Example 4 with SharedMemory

use of org.apache.storm.generated.SharedMemory in project storm by apache.

the class TridentTopologyBuilder method buildTopology.

public StormTopology buildTopology(Map<String, Number> masterCoordResources) {
    TopologyBuilder builder = new TopologyBuilder();
    Map<GlobalStreamId, String> batchIdsForSpouts = fleshOutStreamBatchIds(false);
    Map<GlobalStreamId, String> batchIdsForBolts = fleshOutStreamBatchIds(true);
    Map<String, List<String>> batchesToCommitIds = new HashMap<>();
    Map<String, List<ITridentSpout>> batchesToSpouts = new HashMap<>();
    for (String id : spouts.keySet()) {
        TransactionalSpoutComponent c = spouts.get(id);
        if (c.spout instanceof IRichSpout) {
            // TODO: wrap this to set the stream name
            builder.setSpout(id, (IRichSpout) c.spout, c.parallelism);
        } else {
            String batchGroup = c.batchGroupId;
            if (!batchesToCommitIds.containsKey(batchGroup)) {
                batchesToCommitIds.put(batchGroup, new ArrayList<String>());
            }
            batchesToCommitIds.get(batchGroup).add(c.commitStateId);
            if (!batchesToSpouts.containsKey(batchGroup)) {
                batchesToSpouts.put(batchGroup, new ArrayList<ITridentSpout>());
            }
            batchesToSpouts.get(batchGroup).add((ITridentSpout) c.spout);
            BoltDeclarer scd = builder.setBolt(spoutCoordinator(id), new TridentSpoutCoordinator(c.commitStateId, (ITridentSpout) c.spout)).globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.BATCH_STREAM_ID).globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.SUCCESS_STREAM_ID);
            for (SharedMemory request : c.sharedMemory) {
                scd.addSharedMemory(request);
            }
            scd.addConfigurations(c.componentConf);
            Map<String, TridentBoltExecutor.CoordSpec> specs = new HashMap<>();
            specs.put(c.batchGroupId, new CoordSpec());
            BoltDeclarer bd = builder.setBolt(id, new TridentBoltExecutor(new TridentSpoutExecutor(c.commitStateId, c.streamName, ((ITridentSpout) c.spout)), batchIdsForSpouts, specs), c.parallelism);
            bd.allGrouping(spoutCoordinator(id), MasterBatchCoordinator.BATCH_STREAM_ID);
            bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.SUCCESS_STREAM_ID);
            if (c.spout instanceof ICommitterTridentSpout) {
                bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.COMMIT_STREAM_ID);
            }
            bd.addConfigurations(c.componentConf);
        }
    }
    for (String id : batchPerTupleSpouts.keySet()) {
        SpoutComponent c = batchPerTupleSpouts.get(id);
        SpoutDeclarer d = builder.setSpout(id, new RichSpoutBatchTriggerer((IRichSpout) c.spout, c.streamName, c.batchGroupId), c.parallelism);
        d.addConfigurations(c.componentConf);
    }
    Number onHeap = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
    Number offHeap = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
    Number cpuLoad = masterCoordResources.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
    for (String batch : batchesToCommitIds.keySet()) {
        List<String> commitIds = batchesToCommitIds.get(batch);
        SpoutDeclarer masterCoord = builder.setSpout(masterCoordinator(batch), new MasterBatchCoordinator(commitIds, batchesToSpouts.get(batch)));
        if (onHeap != null) {
            if (offHeap != null) {
                masterCoord.setMemoryLoad(onHeap, offHeap);
            } else {
                masterCoord.setMemoryLoad(onHeap);
            }
        }
        if (cpuLoad != null) {
            masterCoord.setCPULoad(cpuLoad);
        }
    }
    for (String id : bolts.keySet()) {
        Component c = bolts.get(id);
        Map<String, CoordSpec> specs = new HashMap<>();
        for (GlobalStreamId s : getBoltSubscriptionStreams(id)) {
            String batch = batchIdsForBolts.get(s);
            if (!specs.containsKey(batch)) {
                specs.put(batch, new CoordSpec());
            }
            CoordSpec spec = specs.get(batch);
            CoordType ct;
            if (batchPerTupleSpouts.containsKey(s.get_componentId())) {
                ct = CoordType.single();
            } else {
                ct = CoordType.all();
            }
            spec.coords.put(s.get_componentId(), ct);
        }
        for (String b : c.committerBatches) {
            specs.get(b).commitStream = new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID);
        }
        BoltDeclarer d = builder.setBolt(id, new TridentBoltExecutor(c.bolt, batchIdsForBolts, specs), c.parallelism);
        for (SharedMemory request : c.sharedMemory) {
            d.addSharedMemory(request);
        }
        d.addConfigurations(c.componentConf);
        for (InputDeclaration inputDecl : c.declarations) {
            inputDecl.declare(d);
        }
        Map<String, Set<String>> batchToComponents = getBoltBatchToComponentSubscriptions(id);
        for (Map.Entry<String, Set<String>> entry : batchToComponents.entrySet()) {
            for (String comp : entry.getValue()) {
                d.directGrouping(comp, TridentBoltExecutor.coordStream(entry.getKey()));
            }
        }
        for (String b : c.committerBatches) {
            d.allGrouping(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID);
        }
    }
    return builder.createTopology();
}

Example 5 with SharedMemory

use of org.apache.storm.generated.SharedMemory in project storm by apache.

the class Cluster method initializeTopoSharedOffHeapNodeMemoryFlag.

/**
 * Initialize the flag to true if specified topology uses SharedOffHeapNodeMemory, false otherwise.
 *
 * @param td TopologyDetails to examine
 */
private void initializeTopoSharedOffHeapNodeMemoryFlag(TopologyDetails td) {
    String topoId = td.getId();
    topoSharedOffHeapMemoryNodeFlag.put(topoId, false);
    StormTopology topology = td.getTopology();
    if (topology == null) {
        // accommodate multitenant_scheduler_test.clj
        return;
    }
    if (topology.is_set_shared_memory()) {
        for (SharedMemory sharedMemory : topology.get_shared_memory().values()) {
            double val = sharedMemory.get_off_heap_node();
            if (val > 0.0) {
                topoSharedOffHeapMemoryNodeFlag.put(topoId, true);
                return;
            }
        }
    }
}