use of com.datatorrent.stram.stream.BufferServerSubscriber in project apex-core by apache.
the class StreamingContainer method deployInputStreams.
@SuppressWarnings("unchecked")
private void deployInputStreams(List<OperatorDeployInfo> operatorList, HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams) throws UnknownHostException {
/*
* collect any input operators along with their smallest window id,
* those are subsequently used to setup the window generator
*/
ArrayList<OperatorDeployInfo> inputNodes = new ArrayList<>();
long smallestCheckpointedWindowId = Long.MAX_VALUE;
// a simple map which maps the oio node to it's the node which owns the thread.
Map<Integer, Integer> oioNodes = new ConcurrentHashMap<>();
/*
* Hook up all the downstream ports. There are 2 places where we deal with more than 1
* downstream ports. The first one follows immediately for WindowGenerator. The second
* case is when source for the input port of some node in this container is in another
* container. So we need to create the stream. We need to track this stream along with
* other streams,and many such streams may exist, we hash them against buffer server
* info as we did for outputs but throw in the sinkid in the mix as well.
*/
for (OperatorDeployInfo ndi : operatorList) {
if (ndi.inputs == null || ndi.inputs.isEmpty()) {
/*
* This has to be InputNode, so let's hook the WindowGenerator to it.
* A node which does not take any input cannot exist in the DAG since it would be completely
* unaware of the windows. So for that reason, AbstractInputNode allows Component.INPUT port.
*/
inputNodes.add(ndi);
/*
* When we activate the window Generator, we plan to activate it only from required windowId.
*/
ndi.checkpoint = getFinishedCheckpoint(ndi);
if (ndi.checkpoint.windowId < smallestCheckpointedWindowId) {
smallestCheckpointedWindowId = ndi.checkpoint.windowId;
}
} else {
Node<?> node = nodes.get(ndi.id);
for (OperatorDeployInfo.InputDeployInfo nidi : ndi.inputs) {
if (nidi.streamCodecs.size() != 1) {
throw new IllegalStateException("Only one input codec configuration should be present");
}
Map.Entry<Integer, StreamCodec<?>> entry = nidi.streamCodecs.entrySet().iterator().next();
Integer streamCodecIdentifier = entry.getKey();
StreamCodec<?> streamCodec = entry.getValue();
String sourceIdentifier = Integer.toString(nidi.sourceNodeId).concat(Component.CONCAT_SEPARATOR).concat(nidi.sourcePortName);
String sinkIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR).concat(nidi.portName);
int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nidi, ndi);
Checkpoint checkpoint = getFinishedCheckpoint(ndi);
ComponentContextPair<Stream, StreamContext> pair = streams.get(sourceIdentifier);
if (pair == null) {
pair = newStreams.get(sourceIdentifier);
}
if (pair == null) {
/*
* We connect to the buffer server for the input on this port.
* We have already placed all the output streams for all the operators in this container.
* Yet, there is no stream which can source this port so it has to come from the buffer
* server, so let's make a connection to it.
*/
assert (nidi.locality != Locality.CONTAINER_LOCAL && nidi.locality != Locality.THREAD_LOCAL);
StreamContext context = new StreamContext(nidi.declaredStreamId);
context.setBufferServerAddress(InetSocketAddress.createUnresolved(nidi.bufferServerHost, nidi.bufferServerPort));
InetAddress inetAddress = context.getBufferServerAddress().getAddress();
if (inetAddress != null && NetUtils.isLocalAddress(inetAddress)) {
context.setBufferServerAddress(new InetSocketAddress(InetAddress.getByName(null), nidi.bufferServerPort));
}
context.put(StreamContext.BUFFER_SERVER_TOKEN, nidi.bufferServerToken);
String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier;
context.setPortId(nidi.portName);
context.put(StreamContext.CODEC, streamCodec);
context.put(StreamContext.EVENT_LOOP, eventloop);
context.setPartitions(nidi.partitionMask, nidi.partitionKeys);
// context.setSourceId(sourceIdentifier);
context.setSourceId(connIdentifier);
context.setSinkId(sinkIdentifier);
context.setFinishedWindowId(checkpoint.windowId);
BufferServerSubscriber subscriber = fastPublisherSubscriber ? new FastSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":").concat(String.valueOf(nidi.bufferServerPort)).concat("/").concat(connIdentifier), queueCapacity) : new BufferServerSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":").concat(String.valueOf(nidi.bufferServerPort)).concat("/").concat(connIdentifier), queueCapacity);
if (streamCodec instanceof StreamCodecWrapperForPersistance) {
subscriber.acquireReservoirForPersistStream(sinkIdentifier, queueCapacity, streamCodec);
}
SweepableReservoir reservoir = subscriber.acquireReservoir(sinkIdentifier, queueCapacity);
if (checkpoint.windowId >= 0) {
node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId));
}
node.connectInputPort(nidi.portName, reservoir);
newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(subscriber, context));
logger.debug("put input stream {} against key {}", subscriber, sinkIdentifier);
} else {
assert (nidi.locality == Locality.CONTAINER_LOCAL || nidi.locality == Locality.THREAD_LOCAL);
/* we are still dealing with the MuxStream originating at the output of the source port */
StreamContext inlineContext = new StreamContext(nidi.declaredStreamId);
inlineContext.setSourceId(sourceIdentifier);
inlineContext.setSinkId(sinkIdentifier);
Stream stream;
SweepableReservoir reservoir;
switch(nidi.locality) {
case CONTAINER_LOCAL:
int outputQueueCapacity = getOutputQueueCapacity(operatorList, nidi.sourceNodeId, nidi.sourcePortName);
if (outputQueueCapacity > queueCapacity) {
queueCapacity = outputQueueCapacity;
}
stream = new InlineStream(queueCapacity);
reservoir = ((InlineStream) stream).getReservoir();
if (checkpoint.windowId >= 0) {
node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId));
}
break;
case THREAD_LOCAL:
stream = new OiOStream();
reservoir = ((OiOStream) stream).getReservoir();
((OiOStream.OiOReservoir) reservoir).setControlSink(((OiONode) node).getControlSink(reservoir));
oioNodes.put(ndi.id, nidi.sourceNodeId);
break;
default:
throw new IllegalStateException("Locality can be either ContainerLocal or ThreadLocal");
}
node.connectInputPort(nidi.portName, reservoir);
newStreams.put(sinkIdentifier, new ComponentContextPair<>(stream, inlineContext));
if (!(pair.component instanceof Stream.MultiSinkCapableStream)) {
String originalSinkId = pair.context.getSinkId();
/* we come here only if we are trying to augment the dag */
StreamContext muxContext = new StreamContext(nidi.declaredStreamId);
muxContext.setSourceId(sourceIdentifier);
muxContext.setFinishedWindowId(checkpoint.windowId);
muxContext.setSinkId(originalSinkId);
MuxStream muxStream = new MuxStream();
muxStream.setSink(originalSinkId, pair.component);
streams.put(originalSinkId, pair);
Node<?> sourceNode = nodes.get(nidi.sourceNodeId);
sourceNode.connectOutputPort(nidi.sourcePortName, muxStream);
newStreams.put(sourceIdentifier, pair = new ComponentContextPair<Stream, StreamContext>(muxStream, muxContext));
}
/* here everything should be multisink capable */
if (streamCodec instanceof StreamCodecWrapperForPersistance) {
PartitionAwareSinkForPersistence pas;
if (nidi.partitionKeys == null) {
pas = new PartitionAwareSinkForPersistence((StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionMask, stream);
} else {
pas = new PartitionAwareSinkForPersistence((StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream);
}
((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas);
} else if (nidi.partitionKeys == null || nidi.partitionKeys.isEmpty()) {
((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, stream);
} else {
/*
* generally speaking we do not have partitions on the inline streams so the control should not
* come here but if it comes, then we are ready to handle it using the partition aware streams.
*/
PartitionAwareSink<Object> pas = new PartitionAwareSink<>(streamCodec == null ? nonSerializingStreamCodec : (StreamCodec<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream);
((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas);
}
String streamSinkId = pair.context.getSinkId();
if (streamSinkId == null) {
pair.context.setSinkId(sinkIdentifier);
} else {
pair.context.setSinkId(streamSinkId.concat(", ").concat(sinkIdentifier));
}
}
}
}
}
setupOiOGroups(oioNodes);
if (!inputNodes.isEmpty()) {
WindowGenerator windowGenerator = setupWindowGenerator(smallestCheckpointedWindowId);
for (OperatorDeployInfo ndi : inputNodes) {
generators.put(ndi.id, windowGenerator);
Node<?> node = nodes.get(ndi.id);
SweepableReservoir reservoir = windowGenerator.acquireReservoir(String.valueOf(ndi.id), 1024);
if (ndi.checkpoint.windowId >= 0) {
node.connectInputPort(Node.INPUT, new WindowIdActivatedReservoir(Integer.toString(ndi.id), reservoir, ndi.checkpoint.windowId));
}
node.connectInputPort(Node.INPUT, reservoir);
}
}
}
use of com.datatorrent.stram.stream.BufferServerSubscriber in project apex-core by apache.
the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.
@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
final String streamName = "streamName";
final String upstreamNodeId = "upstreamNodeId";
final String downstreamNodeId = "downStreamNodeId";
EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
((DefaultEventLoop) eventloop).start();
// find random port
final Server bufferServer = new Server(eventloop, 0);
final int bufferServerPort = bufferServer.run().getPort();
final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
GenericTestOperator go = new GenericTestOperator();
final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
gn.setId(1);
Sink<Object> output = new Sink<Object>() {
@Override
public void put(Object tuple) {
tuples.add(tuple);
}
@Override
public int getCount(boolean reset) {
return 0;
}
};
InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
StreamContext issContext = new StreamContext(streamName);
issContext.setSourceId(upstreamNodeId);
issContext.setSinkId(downstreamNodeId);
issContext.setFinishedWindowId(-1);
issContext.setBufferServerAddress(socketAddress);
issContext.put(StreamContext.CODEC, serde);
issContext.put(StreamContext.EVENT_LOOP, eventloop);
StreamContext ossContext = new StreamContext(streamName);
ossContext.setSourceId(upstreamNodeId);
ossContext.setSinkId(downstreamNodeId);
ossContext.setBufferServerAddress(socketAddress);
ossContext.put(StreamContext.CODEC, serde);
ossContext.put(StreamContext.EVENT_LOOP, eventloop);
BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
oss.setup(ossContext);
oss.activate(ossContext);
oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
buff[buff.length - 1] = (byte) 1;
oss.put(buff);
oss.put(new EndWindowTuple(0x1L));
oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
buff = PayloadTuple.getSerializedTuple(0, 1);
buff[buff.length - 1] = (byte) 2;
oss.put(buff);
oss.put(new EndWindowTuple(0x2L));
oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
buff = PayloadTuple.getSerializedTuple(0, 1);
buff[buff.length - 1] = (byte) 3;
oss.put(buff);
oss.put(new EndWindowTuple(0x3L));
oss.put(new EndStreamTuple(0L));
BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
iss.setup(issContext);
gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
gn.connectOutputPort(GenericTestOperator.OPORT1, output);
SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
iss.activate(issContext);
while (tupleWait.sweep() == null) {
Thread.sleep(100);
}
gn.firstWindowMillis = 0;
gn.windowWidthMillis = 100;
Thread t = new Thread() {
@Override
public void run() {
gn.activate();
gn.run();
gn.deactivate();
}
};
t.start();
t.join();
Assert.assertEquals(10, tuples.size());
List<Object> list = new ArrayList<>(tuples);
Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
if (bufferServer != null) {
bufferServer.stop();
}
((DefaultEventLoop) eventloop).stop();
}
Aggregations