Examples with Fields org.apache.storm.tuple.Fields used on opensource projects

Example 21 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class SkewedRollingTopWords method run.

/**
   * Submits (runs) the topology.
   *
   * Usage: "SkewedRollingTopWords [topology-name] [-local]"
   *
   * By default, the topology is run locally under the name
   * "slidingWindowCounts".
   *
   * Examples:
   *
   * ```
   *
   * # Runs in local mode (LocalCluster), with topology name "slidingWindowCounts"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords -local
   *
   * # Runs in local mode (LocalCluster), with topology name "foobar"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords foobar -local
   * 
   * # Runs in local mode (LocalCluster) for 30 seconds, with topology name "foobar" 
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords foobar -local -ttl 30
   *
   * # Runs in remote/cluster mode, with topology name "production-topology"
   * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ```
   *
   * @param args
   *          First positional argument (optional) is topology name, second
   *          positional argument (optional) defines whether to run the topology
   *          locally ("-local") or remotely, i.e. on a real cluster.
   * @throws Exception
   */
protected int run(String[] args) {
    String topologyName = "slidingWindowCounts";
    if (args.length >= 1) {
        topologyName = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String aggId = "aggregator";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
    builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
    LOG.info("Topology name: " + topologyName);
    return submit(topologyName, conf, builder);
}

Example 22 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class FluxBuilder method buildStreamDefinitions.

/**
     * @param context
     * @param builder
     */
private static void buildStreamDefinitions(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException {
    TopologyDef topologyDef = context.getTopologyDef();
    // process stream definitions
    HashMap<String, BoltDeclarer> declarers = new HashMap<String, BoltDeclarer>();
    for (StreamDef stream : topologyDef.getStreams()) {
        Object boltObj = context.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IWindowedBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IStatefulBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IStatefulBolt) boltObj, topologyDef.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
        }
        GroupingDef grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping, otherwise assume storm's default stream
        String streamId = (grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId());
        switch(grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                //TODO check for null grouping args
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs()));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case DIRECT:
                declarer.directGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case LOCAL_OR_SHUFFLE:
                declarer.localOrShuffleGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), context));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
}

Example 23 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class TridentMinMaxOfVehiclesTopology method buildVehiclesTopology.

/**
     * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields
     * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link TridentMinMaxOfVehiclesTopology.Driver} respectively.
     */
public static StormTopology buildVehiclesTopology() {
    Fields driverField = new Fields(Driver.FIELD_NAME);
    Fields vehicleField = new Fields(Vehicle.FIELD_NAME);
    Fields allFields = new Fields(Vehicle.FIELD_NAME, Driver.FIELD_NAME);
    FixedBatchSpout spout = new FixedBatchSpout(allFields, 10, Vehicle.generateVehicles(20));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream vehiclesStream = topology.newStream("spout1", spout).each(allFields, new Debug("##### vehicles"));
    Stream slowVehiclesStream = vehiclesStream.min(new SpeedComparator()).each(vehicleField, new Debug("#### slowest vehicle"));
    Stream slowDriversStream = slowVehiclesStream.project(driverField).each(driverField, new Debug("##### slowest driver"));
    vehiclesStream.max(new SpeedComparator()).each(vehicleField, new Debug("#### fastest vehicle")).project(driverField).each(driverField, new Debug("##### fastest driver"));
    vehiclesStream.minBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).each(vehicleField, new Debug("#### least efficient vehicle"));
    vehiclesStream.maxBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).each(vehicleField, new Debug("#### most efficient vehicle"));
    return topology.build();
}

Example 24 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class TridentCalcRel method tridentPlan.

@Override
public void tridentPlan(TridentPlanCreator planCreator) throws Exception {
    // SingleRel
    RelNode input = getInput();
    StormRelUtils.getStormRelInput(input).tridentPlan(planCreator);
    Stream inputStream = planCreator.pop().toStream();
    String stageName = StormRelUtils.getStageName(this);
    RelDataType inputRowType = getInput(0).getRowType();
    List<String> outputFieldNames = getRowType().getFieldNames();
    int outputCount = outputFieldNames.size();
    // filter
    ExecutableExpression filterInstance = null;
    RexLocalRef condition = program.getCondition();
    if (condition != null) {
        RexNode conditionNode = program.expandLocalRef(condition);
        filterInstance = planCreator.createScalarInstance(Lists.newArrayList(conditionNode), inputRowType, StormRelUtils.getClassName(this));
    }
    // projection
    ExecutableExpression projectionInstance = null;
    List<RexLocalRef> projectList = program.getProjectList();
    if (projectList != null && !projectList.isEmpty()) {
        List<RexNode> expandedNodes = new ArrayList<>();
        for (RexLocalRef project : projectList) {
            expandedNodes.add(program.expandLocalRef(project));
        }
        projectionInstance = planCreator.createScalarInstance(expandedNodes, inputRowType, StormRelUtils.getClassName(this));
    }
    if (projectionInstance == null && filterInstance == null) {
        // it shouldn't be happen
        throw new IllegalStateException("Either projection or condition, or both should be provided.");
    }
    final Stream finalStream = inputStream.flatMap(new EvaluationCalc(filterInstance, projectionInstance, outputCount, planCreator.getDataContext()), new Fields(outputFieldNames)).name(stageName);
    planCreator.addStream(finalStream);
}

Example 25 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class TridentStreamInsertRel method tridentPlan.

@Override
public void tridentPlan(TridentPlanCreator planCreator) throws Exception {
    // SingleRel
    RelNode input = getInput();
    StormRelUtils.getStormRelInput(input).tridentPlan(planCreator);
    Stream inputStream = planCreator.pop().toStream();
    String stageName = StormRelUtils.getStageName(this);
    Preconditions.checkArgument(isInsert(), "Only INSERT statement is supported.");
    List<String> inputFields = this.input.getRowType().getFieldNames();
    List<String> outputFields = getRowType().getFieldNames();
    // FIXME: this should be really different...
    String tableName = Joiner.on('.').join(getTable().getQualifiedName());
    ISqlTridentDataSource.SqlTridentConsumer consumer = planCreator.getSources().get(tableName).getConsumer();
    // In fact this is normally the end of stream, but I'm still not sure so I open new streams based on State values
    IAggregatableStream finalStream = inputStream.partitionPersist(consumer.getStateFactory(), new Fields(inputFields), consumer.getStateUpdater(), new Fields(outputFields)).newValuesStream().name(stageName);
    planCreator.addStream(finalStream);
}