Examples with BoltDeclarer org.apache.heron.api.topology.BoltDeclarer used on opensource projects

Example 1 with BoltDeclarer

use of org.apache.heron.api.topology.BoltDeclarer in project heron by twitter.

the class CustomStreamlet method doBuild.

/**
 * Connect this streamlet to TopologyBuilder.
 * @param bldr The TopologyBuilder for the topology
 * @param stageNames The existing stage names
 * @return True if successful
 */
@Override
public boolean doBuild(TopologyBuilder bldr, Set<String> stageNames) {
    // Create and set bolt
    BoltDeclarer declarer;
    if (operator instanceof IStreamletBasicOperator) {
        setDefaultNameIfNone(StreamletNamePrefix.CUSTOM, stageNames);
        IStreamletBasicOperator<R, T> op = (IStreamletBasicOperator<R, T>) operator;
        bldr.setBolt(getName(), op, getNumPartitions()).grouping(parent.getName(), parent.getStreamId(), grouper);
    } else if (operator instanceof IStreamletRichOperator) {
        setDefaultNameIfNone(StreamletNamePrefix.CUSTOM_BASIC, stageNames);
        IStreamletRichOperator<R, T> op = (IStreamletRichOperator<R, T>) operator;
        bldr.setBolt(getName(), op, getNumPartitions()).grouping(parent.getName(), parent.getStreamId(), grouper);
    } else if (operator instanceof IStreamletWindowOperator) {
        setDefaultNameIfNone(StreamletNamePrefix.CUSTOM_WINDOW, stageNames);
        IStreamletWindowOperator<R, T> op = (IStreamletWindowOperator<R, T>) operator;
        bldr.setBolt(getName(), op, getNumPartitions()).grouping(parent.getName(), parent.getStreamId(), grouper);
    } else {
        throw new RuntimeException("Unhandled operator class is found!");
    }
    return true;
}

Example 2 with BoltDeclarer

use of org.apache.heron.api.topology.BoltDeclarer in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 3 with BoltDeclarer

use of org.apache.heron.api.topology.BoltDeclarer in project heron by twitter.

the class StreamBuilder method buildStreams.

protected <K extends Serializable, V extends Serializable> void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder, ObjectBuilder objectBuilder) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
    EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
    Map<String, ComponentStream> componentStreams = new HashMap<>();
    HashMap<String, BoltDeclarer> declarers = new HashMap<>();
    for (StreamDefinition stream : topologyDefinition.getStreams()) {
        Object boltObj = executionContext.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IStatefulWindowedBolt) {
            if (declarer == null) {
                // noinspection unchecked
                declarer = builder.setBolt(stream.getTo(), (IStatefulWindowedBolt<K, V>) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IWindowedBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
        }
        GroupingDefinition grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping,
        // otherwise assume default stream
        String streamId = grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();
        switch(grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                List<String> groupingArgs = grouping.getArgs();
                if (groupingArgs == null) {
                    throw new IllegalArgumentException("You must supply arguments for Fields grouping");
                }
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), executionContext, objectBuilder));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
    executionContext.setStreams(componentStreams);
}