Examples with DefaultAttributeMap com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap used on opensource projects

Example 6 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-malhar by apache.

the class WindowUtilsTest method createOperatorContext.

public static OperatorContext createOperatorContext(int streamingWindowMillis, int appWindowCount) {
    DefaultAttributeMap attributeMap = new DefaultAttributeMap();
    attributeMap.put(DAGContext.STREAMING_WINDOW_SIZE_MILLIS, streamingWindowMillis);
    attributeMap.put(OperatorContext.APPLICATION_WINDOW_COUNT, appWindowCount);
    return mockOperatorContext(1, attributeMap);
}

Example 7 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class GenericNodeTest method testSynchingLogic.

@Test
@SuppressWarnings("SleepWhileInLoop")
public void testSynchingLogic() throws InterruptedException {
    long sleeptime = 25L;
    final ArrayList<Object> list = new ArrayList<>();
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("ip2Res", 1024);
    Sink<Object> output = new Sink<Object>() {

        @Override
        public void put(Object tuple) {
            list.add(tuple);
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    gn.connectInputPort("ip1", reservoir1);
    gn.connectInputPort("ip2", reservoir2);
    gn.connectOutputPort("op", output);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    do {
        Thread.sleep(sleeptime);
    } while (ab.get() == false);
    Tuple beginWindow1 = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    reservoir2.add(beginWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    Tuple endWindow1 = new EndWindowTuple(0x1L);
    reservoir1.add(endWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    Tuple beginWindow2 = new Tuple(MessageType.BEGIN_WINDOW, 0x2L);
    reservoir1.add(beginWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(1, list.size());
    reservoir2.add(endWindow1);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    reservoir2.add(beginWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    Tuple endWindow2 = new EndWindowTuple(0x2L);
    reservoir2.add(endWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(3, list.size());
    reservoir1.add(endWindow2);
    Thread.sleep(sleeptime);
    Assert.assertEquals(4, list.size());
    EndStreamTuple est = new EndStreamTuple(0L);
    reservoir1.add(est);
    Thread.sleep(sleeptime);
    Assert.assertEquals(4, list.size());
    Tuple beginWindow3 = new Tuple(MessageType.BEGIN_WINDOW, 0x3L);
    reservoir2.add(beginWindow3);
    Thread.sleep(sleeptime);
    Assert.assertEquals(5, list.size());
    Tuple endWindow3 = new EndWindowTuple(0x3L);
    reservoir2.add(endWindow3);
    Thread.sleep(sleeptime);
    Assert.assertEquals(6, list.size());
    Assert.assertNotSame(Thread.State.TERMINATED, t.getState());
    reservoir2.add(est);
    Thread.sleep(sleeptime);
    Assert.assertEquals(7, list.size());
    Thread.sleep(sleeptime);
    Assert.assertEquals(Thread.State.TERMINATED, t.getState());
}

Example 8 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class GenericNodeTest method testControlTuplesDeliveryGenericNode.

@Test
public void testControlTuplesDeliveryGenericNode() throws InterruptedException {
    long maxSleep = 5000000;
    long sleeptime = 25L;
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    gn.connectInputPort("ip1", reservoir1);
    TestSink testSink = new TestSink();
    gn.connectOutputPort("op", testSink);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((ab.get() == false) && (interval < maxSleep));
    int controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    controlTupleCount = gn.controlTupleCount;
    CustomControlTuple t1 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(1, false));
    CustomControlTuple t2 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(2, true));
    CustomControlTuple t3 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(3, false));
    CustomControlTuple t4 = new CustomControlTuple(new CustomControlTupleTest.TestControlTuple(4, true));
    reservoir1.add(t1);
    reservoir1.add(t2);
    reservoir1.add(t3);
    reservoir1.add(t4);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("Custom control tuples emitted immediately", testSink.getResultCount() == 3);
    controlTupleCount = gn.controlTupleCount;
    Tuple endWindow = new Tuple(MessageType.END_WINDOW, 0x1L);
    reservoir1.add(endWindow);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    gn.shutdown();
    t.join();
    Assert.assertTrue("Total control tuples", testSink.getResultCount() == 6);
    long expected = 0;
    for (Object o : testSink.collectedTuples) {
        if (o instanceof CustomControlTuple) {
            expected++;
        }
    }
    Assert.assertTrue("Number of Custom control tuples", expected == 4);
}

Example 9 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class GenericNodeTest method testCheckpointDistance.

private void testCheckpointDistance(int dagCheckPoint, int opCheckPoint) throws InterruptedException {
    int windowWidth = 50;
    long sleeptime = 25L;
    int maxWindows = 60;
    // Adding some extra time for the windows to finish
    long maxSleep = windowWidth * maxWindows + 5000;
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1, "default");
    final WindowGenerator windowGenerator = new WindowGenerator(executorService, 1024);
    windowGenerator.setWindowWidth(windowWidth);
    windowGenerator.setFirstWindow(executorService.getCurrentTimeMillis());
    windowGenerator.setCheckpointCount(dagCheckPoint, 0);
    //GenericOperator go = new GenericOperator();
    CheckpointDistanceOperator go = new CheckpointDistanceOperator();
    go.maxWindows = maxWindows;
    List<Integer> checkpoints = new ArrayList<>();
    int window = 0;
    while (window < maxWindows) {
        window = (int) Math.ceil((double) (window + 1) / dagCheckPoint) * dagCheckPoint;
        window = (int) Math.ceil((double) window / opCheckPoint) * opCheckPoint;
        checkpoints.add(window);
    }
    final StreamContext stcontext = new StreamContext("s1");
    DefaultAttributeMap attrMap = new DefaultAttributeMap();
    attrMap.put(Context.DAGContext.CHECKPOINT_WINDOW_COUNT, dagCheckPoint);
    attrMap.put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, opCheckPoint);
    final OperatorContext context = new com.datatorrent.stram.engine.OperatorContext(0, "operator", attrMap, null);
    final GenericNode gn = new GenericNode(go, context);
    gn.setId(1);
    //DefaultReservoir reservoir1 = new DefaultReservoir("ip1Res", 1024);
    //DefaultReservoir reservoir2 = new DefaultReservoir("ip2Res", 1024);
    //gn.connectInputPort("ip1", reservoir1);
    //gn.connectInputPort("ip2", reservoir2);
    gn.connectInputPort("ip1", windowGenerator.acquireReservoir("ip1", 1024));
    gn.connectInputPort("ip2", windowGenerator.acquireReservoir("ip2", 1024));
    gn.connectOutputPort("op", Sink.BLACKHOLE);
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.setup(context);
            windowGenerator.activate(stcontext);
            gn.activate();
            ab.set(true);
            gn.run();
            windowGenerator.deactivate();
            gn.deactivate();
            gn.teardown();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((go.numWindows < maxWindows) && (interval < maxSleep));
    Assert.assertEquals("Number distances", maxWindows, go.numWindows);
    int chkindex = 0;
    int nextCheckpoint = checkpoints.get(chkindex++);
    for (int i = 0; i < maxWindows; ++i) {
        if ((i + 1) > nextCheckpoint) {
            nextCheckpoint = checkpoints.get(chkindex++);
        }
        Assert.assertEquals("Windows from checkpoint for " + i, nextCheckpoint - i, (int) go.distances.get(i));
    }
    gn.shutdown();
    t.join();
}

Example 10 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class GenericNodeTest method testPrematureTermination.

@Test
public void testPrematureTermination() throws InterruptedException {
    long maxSleep = 5000;
    long sleeptime = 25L;
    GenericOperator go = new GenericOperator();
    final GenericNode gn = new GenericNode(go, new com.datatorrent.stram.engine.OperatorContext(0, "operator", new DefaultAttributeMap(), null));
    gn.setId(1);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("ip1Res", 1024);
    AbstractReservoir reservoir2 = AbstractReservoir.newReservoir("ip2Res", 1024);
    gn.connectInputPort("ip1", reservoir1);
    gn.connectInputPort("ip2", reservoir2);
    gn.connectOutputPort("op", Sink.BLACKHOLE);
    gn.firstWindowMillis = 0;
    gn.windowWidthMillis = 100;
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            ab.set(true);
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((ab.get() == false) && (interval < maxSleep));
    int controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow1 = new Tuple(MessageType.BEGIN_WINDOW, 0x1L);
    reservoir1.add(beginWindow1);
    reservoir2.add(beginWindow1);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("Begin window called", go.endWindowId != go.beginWindowId);
    controlTupleCount = gn.controlTupleCount;
    Tuple endWindow1 = new EndWindowTuple(0x1L);
    reservoir1.add(endWindow1);
    reservoir2.add(endWindow1);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    Assert.assertTrue("End window called", go.endWindowId == go.beginWindowId);
    controlTupleCount = gn.controlTupleCount;
    Tuple beginWindow2 = new Tuple(MessageType.BEGIN_WINDOW, 0x2L);
    reservoir1.add(beginWindow2);
    reservoir2.add(beginWindow2);
    interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((gn.controlTupleCount == controlTupleCount) && (interval < maxSleep));
    gn.shutdown();
    t.join();
    Assert.assertTrue("End window not called", go.endWindowId != go.beginWindowId);
}