Examples with TridentTuple org.apache.storm.trident.tuple.TridentTuple used on opensource projects

Example 41 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class SampleDruidBoltTridentTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length == 0) {
        throw new IllegalArgumentException("There should be at least one argument. Run as `SampleDruidBoltTridentTopology <zk-url>`");
    }
    TridentTopology tridentTopology = new TridentTopology();
    DruidBeamFactory druidBeamFactory = new SampleDruidBeamFactoryImpl(new HashMap<String, Object>());
    ITupleDruidEventMapper<Map<String, Object>> eventMapper = new TupleDruidEventMapper<>(TupleDruidEventMapper.DEFAULT_FIELD_NAME);
    final Stream stream = tridentTopology.newStream("batch-event-gen", new SimpleBatchSpout(10));
    stream.peek(new Consumer() {

        @Override
        public void accept(TridentTuple input) {
            LOG.info("########### Received tuple: [{}]", input);
        }
    }).partitionPersist(new DruidBeamStateFactory<Map<String, Object>>(druidBeamFactory, eventMapper), new Fields("event"), new DruidBeamStateUpdater());
    Config conf = new Config();
    conf.setDebug(true);
    conf.put("druid.tranquility.zk.connect", args[0]);
    if (args.length > 1) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[1], conf, tridentTopology.build());
    } else {
        conf.setMaxTaskParallelism(3);
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("druid-test", conf, tridentTopology.build())) {
            Thread.sleep(30000);
        }
        System.exit(0);
    }
}

Example 42 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class EsState method updateState.

/**
     * Store current state to ElasticSearch.
     *
     * @param tuples list of tuples for storing to ES.
     *               Each tuple should have relevant fields (source, index, type, id) for EsState's tupleMapper to extract ES document.
     */
public void updateState(List<TridentTuple> tuples) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (TridentTuple tuple : tuples) {
        String source = tupleMapper.getSource(tuple);
        String index = tupleMapper.getIndex(tuple);
        String type = tupleMapper.getType(tuple);
        String id = tupleMapper.getId(tuple);
        bulkRequest.add(client.prepareIndex(index, type, id).setSource(source));
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        LOG.warn("failed processing bulk index requests " + bulkResponse.buildFailureMessage());
        throw new FailedException();
    }
}

Example 43 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class CassandraState method updateState.

public void updateState(List<TridentTuple> tuples, final TridentCollector collector) {
    List<Statement> statements = new ArrayList<>();
    for (TridentTuple tuple : tuples) {
        statements.addAll(options.cqlStatementTupleMapper.map(conf, session, tuple));
    }
    try {
        if (options.batchingType != null) {
            BatchStatement batchStatement = new BatchStatement(options.batchingType);
            batchStatement.addAll(statements);
            session.execute(batchStatement);
        } else {
            for (Statement statement : statements) {
                session.execute(statement);
            }
        }
    } catch (Exception e) {
        LOG.warn("Batch write operation is failed.");
        collector.reportError(e);
        throw new FailedException(e);
    }
}

Example 44 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class CassandraState method batchRetrieve.

public List<List<Values>> batchRetrieve(List<TridentTuple> tridentTuples) {
    Preconditions.checkNotNull(options.cqlResultSetValuesMapper, "CassandraState.Options should have cqlResultSetValuesMapper");
    List<List<Values>> batchRetrieveResult = new ArrayList<>();
    try {
        for (TridentTuple tridentTuple : tridentTuples) {
            List<Statement> statements = options.cqlStatementTupleMapper.map(conf, session, tridentTuple);
            for (Statement statement : statements) {
                List<List<Values>> values = options.cqlResultSetValuesMapper.map(session, statement, tridentTuple);
                batchRetrieveResult.addAll(values);
            }
        }
    } catch (Exception e) {
        LOG.warn("Batch retrieve operation is failed.");
        throw new FailedException(e);
    }
    return batchRetrieveResult;
}

Example 45 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class TestSocketDataSourceProvider method mockTupleList.

private static List<TridentTuple> mockTupleList() {
    List<TridentTuple> tupleList = new ArrayList<>();
    TridentTuple t0 = mock(TridentTuple.class);
    TridentTuple t1 = mock(TridentTuple.class);
    when(t0.getValueByField("ID")).thenReturn(1);
    when(t0.getValueByField("val")).thenReturn("2");
    doReturn(Lists.<Object>newArrayList(1, "2")).when(t0).getValues();
    when(t0.size()).thenReturn(2);
    when(t1.getValueByField("ID")).thenReturn(2);
    when(t1.getValueByField("val")).thenReturn("3");
    doReturn(Lists.<Object>newArrayList(2, "3")).when(t1).getValues();
    when(t1.size()).thenReturn(2);
    tupleList.add(t0);
    tupleList.add(t1);
    return tupleList;
}