Examples with TridentTuple org.apache.storm.trident.tuple.TridentTuple used on opensource projects

Example 36 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class MongoState method updateState.

public void updateState(List<TridentTuple> tuples, TridentCollector collector) {
    List<Document> documents = Lists.newArrayList();
    for (TridentTuple tuple : tuples) {
        Document document = options.mapper.toDocument(tuple);
        documents.add(document);
    }
    try {
        this.mongoClient.insert(documents, true);
    } catch (Exception e) {
        LOG.warn("Batch write failed but some requests might have succeeded. Triggering replay.", e);
        throw new FailedException(e);
    }
}

Example 37 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class TestRedisDataSourcesProvider method testRedisClusterSink.

@SuppressWarnings("unchecked")
@Test
public void testRedisClusterSink() throws IOException {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("redis://localhost:6380"), null, null, CLUSTER_TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(RedisClusterState.Factory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(RedisClusterStateUpdater.class, consumer.getStateUpdater().getClass());
    RedisClusterState state = (RedisClusterState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
    StateUpdater stateUpdater = consumer.getStateUpdater();
    JedisCluster mockJedisCluster = mock(JedisCluster.class);
    Whitebox.setInternalState(state, "jedisCluster", mockJedisCluster);
    List<TridentTuple> tupleList = mockTupleList();
    stateUpdater.updateState(state, tupleList, null);
    for (TridentTuple t : tupleList) {
        // PK goes to the key
        String id = String.valueOf(t.getValueByField("ID"));
        String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
        verify(mockJedisCluster).hset(eq(ADDITIONAL_KEY), eq(id), eq(serializedValue));
    }
    verify(mockJedisCluster, never()).close();
}

Example 38 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class TestRedisDataSourcesProvider method testRedisSink.

@SuppressWarnings("unchecked")
@Test
public void testRedisSink() {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("redis://:foobared@localhost:6380/2"), null, null, TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(RedisState.Factory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(RedisStateUpdater.class, consumer.getStateUpdater().getClass());
    RedisState state = (RedisState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
    StateUpdater stateUpdater = consumer.getStateUpdater();
    JedisPool mockJedisPool = mock(JedisPool.class);
    Jedis mockJedis = mock(Jedis.class);
    Pipeline mockPipeline = mock(Pipeline.class);
    Whitebox.setInternalState(state, "jedisPool", mockJedisPool);
    when(mockJedisPool.getResource()).thenReturn(mockJedis);
    when(mockJedis.pipelined()).thenReturn(mockPipeline);
    List<TridentTuple> tupleList = mockTupleList();
    stateUpdater.updateState(state, tupleList, null);
    for (TridentTuple t : tupleList) {
        // PK goes to the key
        String id = String.valueOf(t.getValueByField("ID"));
        String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
        verify(mockPipeline).hset(eq(ADDITIONAL_KEY), eq(id), eq(serializedValue));
    }
    verify(mockPipeline).sync();
    verify(mockJedis).close();
}

Example 39 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class TestKafkaDataSourcesProvider method testKafkaSink.

@SuppressWarnings("unchecked")
@Test
public void testKafkaSink() {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("kafka://mock?topic=foo"), null, null, TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(TridentKafkaStateFactory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(TridentKafkaUpdater.class, consumer.getStateUpdater().getClass());
    TridentKafkaState state = (TridentKafkaState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
    KafkaProducer producer = mock(KafkaProducer.class);
    doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class));
    Whitebox.setInternalState(state, "producer", producer);
    List<TridentTuple> tupleList = mockTupleList();
    for (TridentTuple t : tupleList) {
        state.updateState(Collections.singletonList(t), null);
        verify(producer).send(argThat(new KafkaMessageMatcher(t)));
    }
    verifyNoMoreInteractions(producer);
}

Example 40 with TridentTuple

use of org.apache.storm.trident.tuple.TridentTuple in project storm by apache.

the class TestMongoDataSourcesProvider method testMongoSink.

@SuppressWarnings("unchecked")
@Test
public void testMongoSink() {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("mongodb://127.0.0.1:27017/test"), null, null, TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(MongoStateFactory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(MongoStateUpdater.class, consumer.getStateUpdater().getClass());
    MongoState state = (MongoState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
    StateUpdater stateUpdater = consumer.getStateUpdater();
    MongoDBClient mongoClient = mock(MongoDBClient.class);
    Whitebox.setInternalState(state, "mongoClient", mongoClient);
    List<TridentTuple> tupleList = mockTupleList();
    for (TridentTuple t : tupleList) {
        stateUpdater.updateState(state, Collections.singletonList(t), null);
        verify(mongoClient).insert(argThat(new MongoArgMatcher(t)), eq(true));
    }
    verifyNoMoreInteractions(mongoClient);
}