Examples with Sink com.datatorrent.api.Sink used on opensource projects

Example 11 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class GenericNodeTest method testControlTuplesDeliveryOiONode.

@Test
public void testControlTuplesDeliveryOiONode() throws InterruptedException {
    GenericOperator go = new GenericOperator();
    final OiONode oioNode = new OiONode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    oioNode.setId(1);
    OiOStream stream = new OiOStream();
    SweepableReservoir reservoir = stream.getReservoir();
    ((OiOStream.OiOReservoir) reservoir).setControlSink((oioNode).getControlSink(reservoir));
    oioNode.connectInputPort("ip1", reservoir);
    Sink controlSink = oioNode.getControlSink(reservoir);
    TestSink testSink = new TestSink();
    oioNode.connectOutputPort("op", testSink);
    oioNode.firstWindowMillis = 0;
    oioNode.windowWidthMillis = 100;
    oioNode.activate();
    Tuple beginWindow = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    controlSink.put(beginWindow);
    Assert.assertTrue("Begin window", testSink.getResultCount() == 1);
    CustomControlTuple t1 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(1, false));
    CustomControlTuple t2 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(2, true));
    CustomControlTuple t3 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(3, false));
    CustomControlTuple t4 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(4, true));
    controlSink.put(t1);
    controlSink.put(t2);
    controlSink.put(t3);
    controlSink.put(t4);
    Assert.assertTrue("Custom control tuples emitted immediately", testSink.getResultCount() == 3);
    Tuple endWindow = new Tuple(MessageType.END_WINDOW, 0x1L);
    controlSink.put(endWindow);
    oioNode.deactivate();
    oioNode.shutdown();
    Assert.assertTrue("Total control tuples", testSink.getResultCount() == 6);
    long expected = 0;
    for (Object o : testSink.collectedTuples) {
        if (o instanceof CustomControlTuple) {
            expected++;
        }
    }
    Assert.assertTrue("Number of Custom control tuples", expected == 4);
}

Example 12 with Sink

use of com.datatorrent.api.Sink in project apex-core by apache.

the class GenericNodeTest method testBufferServerSubscriberActivationBeforeOperator.

@Test
public void testBufferServerSubscriberActivationBeforeOperator() throws InterruptedException, IOException {
    final String streamName = "streamName";
    final String upstreamNodeId = "upstreamNodeId";
    final String downstreamNodeId = "downStreamNodeId";
    EventLoop eventloop = DefaultEventLoop.createEventLoop("StreamTestEventLoop");
    ((DefaultEventLoop) eventloop).start();
    // find random port
    final Server bufferServer = new Server(eventloop, 0);
    final int bufferServerPort = bufferServer.run().getPort();
    final StreamCodec<Object> serde = new DefaultStatefulStreamCodec<>();
    final BlockingQueue<Object> tuples = new ArrayBlockingQueue<>(10);
    GenericTestOperator go = new GenericTestOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            tuples.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    InetSocketAddress socketAddress = new InetSocketAddress("localhost", bufferServerPort);
    StreamContext issContext = new StreamContext(streamName);
    issContext.setSourceId(upstreamNodeId);
    issContext.setSinkId(downstreamNodeId);
    issContext.setFinishedWindowId(-1);
    issContext.setBufferServerAddress(socketAddress);
    issContext.put(StreamContext.CODEC, serde);
    issContext.put(StreamContext.EVENT_LOOP, eventloop);
    StreamContext ossContext = new StreamContext(streamName);
    ossContext.setSourceId(upstreamNodeId);
    ossContext.setSinkId(downstreamNodeId);
    ossContext.setBufferServerAddress(socketAddress);
    ossContext.put(StreamContext.CODEC, serde);
    ossContext.put(StreamContext.EVENT_LOOP, eventloop);
    BufferServerPublisher oss = new BufferServerPublisher(upstreamNodeId, 1024);
    oss.setup(ossContext);
    oss.activate(ossContext);
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x1L));
    byte[] buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 1;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x1L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x2L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 2;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x2L));
    oss.put(new Tuple(MessageType.BEGIN_WINDOW, 0x3L));
    buff = PayloadTuple.getSerializedTuple(0, 1);
    buff[buff.length - 1] = (byte) 3;
    oss.put(buff);
    oss.put(new EndWindowTuple(0x3L));
    oss.put(new EndStreamTuple(0L));
    BufferServerSubscriber iss = new BufferServerSubscriber(downstreamNodeId, 1024);
    iss.setup(issContext);
    gn.connectInputPort(GenericTestOperator.IPORT1, iss.acquireReservoir("testReservoir", 10));
    gn.connectOutputPort(GenericTestOperator.OPORT1, output);
    SweepableReservoir tupleWait = iss.acquireReservoir("testReservoir2", 10);
    iss.activate(issContext);
    while (tupleWait.sweep() == null) {
        Thread.sleep(100);
    }
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    t.join();
    Assert.assertEquals(10, tuples.size());
    List<Object> list = new ArrayList<>(tuples);
    Assert.assertEquals("Payload Tuple 1", 1, ((byte[]) list.get(1))[5]);
    Assert.assertEquals("Payload Tuple 2", 2, ((byte[]) list.get(4))[5]);
    Assert.assertEquals("Payload Tuple 3", 3, ((byte[]) list.get(7))[5]);
    if (bufferServer != null) {
        bufferServer.stop();
    }
    ((DefaultEventLoop) eventloop).stop();
}

Example 13 with Sink

use of com.datatorrent.api.Sink in project beam by apache.

the class ParDoTranslatorTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class);
    options.setRunner(TestApexRunner.class);
    Pipeline pipeline = Pipeline.create(options);
    Coder<WindowedValue<Integer>> coder = WindowedValue.getValueOnlyCoder(VarIntCoder.of());
    PCollectionView<Integer> singletonView = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView());
    ApexParDoOperator<Integer, Integer> operator = new ApexParDoOperator<>(options, new Add(singletonView), new TupleTag<Integer>(), TupleTagList.empty().getAll(), WindowingStrategy.globalDefault(), Collections.<PCollectionView<?>>singletonList(singletonView), coder, new ApexStateInternals.ApexStateBackend());
    operator.setup(null);
    operator.beginWindow(0);
    WindowedValue<Integer> wv1 = WindowedValue.valueInGlobalWindow(1);
    WindowedValue<Iterable<?>> sideInput = WindowedValue.<Iterable<?>>valueInGlobalWindow(Lists.<Integer>newArrayList(22));
    // pushed back input
    operator.input.process(ApexStreamTuple.DataTuple.of(wv1));
    final List<Object> results = Lists.newArrayList();
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            results.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    // verify pushed back input checkpointing
    Assert.assertNotNull("Serialization", operator = KryoCloneUtils.cloneObject(operator));
    operator.output.setSink(sink);
    operator.setup(null);
    operator.beginWindow(1);
    WindowedValue<Integer> wv2 = WindowedValue.valueInGlobalWindow(2);
    operator.sideInput1.process(ApexStreamTuple.DataTuple.of(sideInput));
    Assert.assertEquals("number outputs", 1, results.size());
    Assert.assertEquals("result", WindowedValue.valueInGlobalWindow(23), ((ApexStreamTuple.DataTuple<?>) results.get(0)).getValue());
    // verify side input checkpointing
    results.clear();
    Assert.assertNotNull("Serialization", operator = KryoCloneUtils.cloneObject(operator));
    operator.output.setSink(sink);
    operator.setup(null);
    operator.beginWindow(2);
    operator.input.process(ApexStreamTuple.DataTuple.of(wv2));
    Assert.assertEquals("number outputs", 1, results.size());
    Assert.assertEquals("result", WindowedValue.valueInGlobalWindow(24), ((ApexStreamTuple.DataTuple<?>) results.get(0)).getValue());
}

Example 14 with Sink

use of com.datatorrent.api.Sink in project beam by apache.

the class ApexGroupByKeyOperatorTest method testGlobalWindowMinTimestamp.

@Test
public void testGlobalWindowMinTimestamp() throws Exception {
    ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class);
    options.setRunner(TestApexRunner.class);
    Pipeline pipeline = Pipeline.create(options);
    WindowingStrategy<?, ?> ws = WindowingStrategy.of(FixedWindows.of(Duration.standardSeconds(10)));
    PCollection<KV<String, Integer>> input = PCollection.createPrimitiveOutputInternal(pipeline, ws, IsBounded.BOUNDED);
    input.setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
    ApexGroupByKeyOperator<String, Integer> operator = new ApexGroupByKeyOperator<>(options, input, new ApexStateInternals.ApexStateBackend());
    operator.setup(null);
    operator.beginWindow(1);
    Assert.assertNotNull("Serialization", operator = KryoCloneUtils.cloneObject(operator));
    final List<Object> results = Lists.newArrayList();
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            results.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    operator.output.setSink(sink);
    operator.setup(null);
    operator.beginWindow(1);
    Instant windowStart = BoundedWindow.TIMESTAMP_MIN_VALUE;
    BoundedWindow window = new IntervalWindow(windowStart, windowStart.plus(10000));
    PaneInfo paneInfo = PaneInfo.NO_FIRING;
    WindowedValue<KV<String, Integer>> wv1 = WindowedValue.of(KV.of("foo", 1), windowStart, window, paneInfo);
    operator.input.process(ApexStreamTuple.DataTuple.of(wv1));
    WindowedValue<KV<String, Integer>> wv2 = WindowedValue.of(KV.of("foo", 1), windowStart, window, paneInfo);
    operator.input.process(ApexStreamTuple.DataTuple.of(wv2));
    ApexStreamTuple<WindowedValue<KV<String, Integer>>> watermark = ApexStreamTuple.WatermarkTuple.of(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis());
    Assert.assertEquals("number outputs", 0, results.size());
    operator.input.process(watermark);
    Assert.assertEquals("number outputs", 2, results.size());
    @SuppressWarnings({ "unchecked", "rawtypes" }) ApexStreamTuple.DataTuple<WindowedValue<KV<String, Iterable<Integer>>>> dataTuple = (ApexStreamTuple.DataTuple) results.get(0);
    List<Integer> counts = Lists.newArrayList(1, 1);
    Assert.assertEquals("iterable", KV.of("foo", counts), dataTuple.getValue().getValue());
    Assert.assertEquals("expected watermark", watermark, results.get(1));
}

Example 15 with Sink

use of com.datatorrent.api.Sink in project apex-malhar by apache.

the class InvertIndexTest method testNodeProcessing.

/**
 * Test oper logic emits correct results
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testNodeProcessing() throws Exception {
    InvertIndex<String, String> oper = new InvertIndex<String, String>();
    CollectorTestSink indexSink = new CollectorTestSink();
    Sink inSink = oper.data.getSink();
    oper.index.setSink(indexSink);
    oper.beginWindow(0);
    HashMap<String, String> input = new HashMap<String, String>();
    input.put("a", "str");
    input.put("b", "str");
    inSink.put(input);
    input.clear();
    input.put("a", "str1");
    input.put("b", "str1");
    inSink.put(input);
    input.clear();
    input.put("c", "blah");
    inSink.put(input);
    input.clear();
    input.put("c", "str1");
    inSink.put(input);
    oper.endWindow();
    Assert.assertEquals("number emitted tuples", 3, indexSink.collectedTuples.size());
    for (Object o : indexSink.collectedTuples) {
        log.debug(o.toString());
        HashMap<String, ArrayList<String>> output = (HashMap<String, ArrayList<String>>) o;
        for (Map.Entry<String, ArrayList<String>> e : output.entrySet()) {
            String key = e.getKey();
            ArrayList<String> alist = e.getValue();
            if (key.equals("str1")) {
                Assert.assertEquals("Index for \"str1\" contains \"a\"", true, alist.contains("a"));
                Assert.assertEquals("Index for \"str1\" contains \"b\"", true, alist.contains("b"));
                Assert.assertEquals("Index for \"str1\" contains \"c\"", true, alist.contains("c"));
            } else if (key.equals("str")) {
                Assert.assertEquals("Index for \"str1\" contains \"a\"", true, alist.contains("a"));
                Assert.assertEquals("Index for \"str1\" contains \"b\"", true, alist.contains("b"));
                Assert.assertEquals("Index for \"str1\" contains \"c\"", false, alist.contains("c"));
            } else if (key.equals("blah")) {
                Assert.assertEquals("Index for \"str1\" contains \"a\"", false, alist.contains("a"));
                Assert.assertEquals("Index for \"str1\" contains \"b\"", false, alist.contains("b"));
                Assert.assertEquals("Index for \"str1\" contains \"c\"", true, alist.contains("c"));
            }
        }
    }
}