use of org.apache.storm.trident.state.StateUpdater in project storm by apache.
the class TestHdfsDataSourcesProvider method testHdfsSink.
@SuppressWarnings("unchecked")
@Test
public void testHdfsSink() {
ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create(hdfsURI), null, null, TBL_PROPERTIES, FIELDS);
Assert.assertNotNull(ds);
ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
Assert.assertEquals(HdfsStateFactory.class, consumer.getStateFactory().getClass());
Assert.assertEquals(HdfsUpdater.class, consumer.getStateUpdater().getClass());
HdfsState state = (HdfsState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
StateUpdater stateUpdater = consumer.getStateUpdater();
HdfsFileOptions options = mock(HdfsFileOptions.class);
Whitebox.setInternalState(state, "options", options);
List<TridentTuple> tupleList = mockTupleList();
for (TridentTuple t : tupleList) {
stateUpdater.updateState(state, Collections.singletonList(t), null);
try {
verify(options).execute(Collections.singletonList(t));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
use of org.apache.storm.trident.state.StateUpdater in project storm by apache.
the class TestSocketDataSourceProvider method testSocketSink.
@Test
public void testSocketSink() throws IOException {
ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("socket://localhost:8888"), null, null, new Properties(), FIELDS);
Assert.assertNotNull(ds);
ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
Assert.assertEquals(SocketState.Factory.class, consumer.getStateFactory().getClass());
Assert.assertEquals(SocketStateUpdater.class, consumer.getStateUpdater().getClass());
// makeState() fails on creating State so we just mock SocketState anyway
SocketState mockState = mock(SocketState.class);
StateUpdater stateUpdater = consumer.getStateUpdater();
List<TridentTuple> tupleList = mockTupleList();
stateUpdater.updateState(mockState, tupleList, null);
for (TridentTuple t : tupleList) {
String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
verify(mockState).write(serializedValue + "\n");
}
}
use of org.apache.storm.trident.state.StateUpdater in project storm by apache.
the class Stream method window.
private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields, boolean storeTuplesInStore) {
projectionValidation(inputFields);
windowConfig.validate();
Fields fields = addTriggerField(functionFields);
// when storeTuplesInStore is false then the given windowStoreFactory is only used to store triggers and
// that store is passed to WindowStateUpdater to remove them after committing the batch.
Stream stream = _topology.addSourcedNode(this, new ProcessorNode(_topology.getUniqueStreamId(), _name, fields, fields, new WindowTridentProcessor(windowConfig, _topology.getUniqueWindowId(), windowStoreFactory, inputFields, aggregator, storeTuplesInStore)));
Stream effectiveStream = stream.project(functionFields);
// create StateUpdater with the given windowStoreFactory to remove triggered aggregation results form store
// when they are successfully processed.
StateFactory stateFactory = new WindowsStateFactory();
StateUpdater stateUpdater = new WindowsStateUpdater(windowStoreFactory);
stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), stateUpdater, new Fields());
return effectiveStream;
}
use of org.apache.storm.trident.state.StateUpdater in project storm by apache.
the class TestRedisDataSourcesProvider method testRedisClusterSink.
@SuppressWarnings("unchecked")
@Test
public void testRedisClusterSink() throws IOException {
ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("redis://localhost:6380"), null, null, CLUSTER_TBL_PROPERTIES, FIELDS);
Assert.assertNotNull(ds);
ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
Assert.assertEquals(RedisClusterState.Factory.class, consumer.getStateFactory().getClass());
Assert.assertEquals(RedisClusterStateUpdater.class, consumer.getStateUpdater().getClass());
RedisClusterState state = (RedisClusterState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
StateUpdater stateUpdater = consumer.getStateUpdater();
JedisCluster mockJedisCluster = mock(JedisCluster.class);
Whitebox.setInternalState(state, "jedisCluster", mockJedisCluster);
List<TridentTuple> tupleList = mockTupleList();
stateUpdater.updateState(state, tupleList, null);
for (TridentTuple t : tupleList) {
// PK goes to the key
String id = String.valueOf(t.getValueByField("ID"));
String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
verify(mockJedisCluster).hset(eq(ADDITIONAL_KEY), eq(id), eq(serializedValue));
}
verify(mockJedisCluster, never()).close();
}
use of org.apache.storm.trident.state.StateUpdater in project storm by apache.
the class TestRedisDataSourcesProvider method testRedisSink.
@SuppressWarnings("unchecked")
@Test
public void testRedisSink() {
ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("redis://:foobared@localhost:6380/2"), null, null, TBL_PROPERTIES, FIELDS);
Assert.assertNotNull(ds);
ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
Assert.assertEquals(RedisState.Factory.class, consumer.getStateFactory().getClass());
Assert.assertEquals(RedisStateUpdater.class, consumer.getStateUpdater().getClass());
RedisState state = (RedisState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
StateUpdater stateUpdater = consumer.getStateUpdater();
JedisPool mockJedisPool = mock(JedisPool.class);
Jedis mockJedis = mock(Jedis.class);
Pipeline mockPipeline = mock(Pipeline.class);
Whitebox.setInternalState(state, "jedisPool", mockJedisPool);
when(mockJedisPool.getResource()).thenReturn(mockJedis);
when(mockJedis.pipelined()).thenReturn(mockPipeline);
List<TridentTuple> tupleList = mockTupleList();
stateUpdater.updateState(state, tupleList, null);
for (TridentTuple t : tupleList) {
// PK goes to the key
String id = String.valueOf(t.getValueByField("ID"));
String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
verify(mockPipeline).hset(eq(ADDITIONAL_KEY), eq(id), eq(serializedValue));
}
verify(mockPipeline).sync();
verify(mockJedis).close();
}
Aggregations