Examples with Fields org.apache.heron.api.tuple.Fields used on opensource projects

Example 16 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class WindowedWordCountTopology method main.

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    int parallelism = 1;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("sentence", new SentenceSpout(), parallelism);
    builder.setBolt("split", new SplitSentence(), parallelism).shuffleGrouping("sentence");
    builder.setBolt("consumer", new WindowSumBolt().withWindow(BaseWindowedBolt.Count.of(10000), BaseWindowedBolt.Count.of(5000)), parallelism).fieldsGrouping("split", new Fields("word"));
    Config conf = new Config();
    conf.setMaxSpoutPending(1000000);
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 17 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class WindowedBoltExecutor method declareOutputFields.

@Override
@SuppressWarnings("HiddenField")
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    String lateTupleStream = (String) getComponentConfiguration().get(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM);
    if (lateTupleStream != null) {
        declarer.declareStream(lateTupleStream, new Fields(LATE_TUPLE_FIELD));
    }
    bolt.declareOutputFields(declarer);
}

Example 18 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class TopologyManagerTest method getTestTopology.

/**
 * Construct the test topology
 */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields(STREAM_ID));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping(STREAM_ID);
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
    conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 19 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class StreamBuilder method buildStreams.

protected <K extends Serializable, V extends Serializable> void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder, ObjectBuilder objectBuilder) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
    EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
    Map<String, ComponentStream> componentStreams = new HashMap<>();
    HashMap<String, BoltDeclarer> declarers = new HashMap<>();
    for (StreamDefinition stream : topologyDefinition.getStreams()) {
        Object boltObj = executionContext.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IStatefulWindowedBolt) {
            if (declarer == null) {
                // noinspection unchecked
                declarer = builder.setBolt(stream.getTo(), (IStatefulWindowedBolt<K, V>) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IWindowedBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
        }
        GroupingDefinition grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping,
        // otherwise assume default stream
        String streamId = grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();
        switch(grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                List<String> groupingArgs = grouping.getArgs();
                if (groupingArgs == null) {
                    throw new IllegalArgumentException("You must supply arguments for Fields grouping");
                }
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), executionContext, objectBuilder));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
    executionContext.setStreams(componentStreams);
}

Example 20 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class GeneralTopologyContextImpl method getOutputToComponentsFields.

public static Map<String, Map<String, Fields>> getOutputToComponentsFields(List<TopologyAPI.OutputStream> outputs) {
    Map<String, Map<String, Fields>> outputFields = new HashMap<>();
    for (TopologyAPI.OutputStream outputStream : outputs) {
        String componentName = outputStream.getStream().getComponentName();
        String streamId = outputStream.getStream().getId();
        Map<String, Fields> componentFields = outputFields.get(componentName);
        if (componentFields == null) {
            componentFields = new HashMap<>();
        }
        // Get the fields of a particular OutputStream
        List<String> retval = new ArrayList<>();
        for (TopologyAPI.StreamSchema.KeyType kt : outputStream.getSchema().getKeysList()) {
            retval.add(kt.getKey());
        }
        // Put it into the map
        componentFields.put(streamId, new Fields(retval));
        outputFields.put(componentName, componentFields);
    }
    return outputFields;
}