Examples with DefaultAttributeMap com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap used on opensource projects

Example 11 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class GenericNodeTest method testCheckpointApplicationWindowCount.

private void testCheckpointApplicationWindowCount(ProcessingMode processingMode) throws Exception {
    final long timeoutMillis = 10000L;
    final long sleepTime = 25L;
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    long resetWindow = 0L;
    long firstWindowMillis = 1448909287863L;
    int windowWidth = 100;
    windowGenerator.setResetWindow(resetWindow);
    windowGenerator.setFirstWindow(firstWindowMillis);
    windowGenerator.setWindowWidth(windowWidth);
    windowGenerator.setCheckpointCount(1, 0);
    GenericOperator go = new GenericOperator();
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(OperatorContext.APPLICATION_WINDOW_COUNT, 5);
    dam.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 5);
    dam.put(OperatorContext.PROCESSING_MODE, processingMode);
    DelayAsyncFSStorageAgent storageAgent = new DelayAsyncFSStorageAgent(testMeta.getDir(), new Configuration());
    storageAgent.setDelayMS(200L);
    dam.put(OperatorContext.STORAGE_AGENT, storageAgent);
    TestStatsOperatorContext operatorContext = new TestStatsOperatorContext(0, "operator", dam, null);
    final GenericNode gn = new GenericNode(go, operatorContext);
    gn.setId(1);
    TestSink testSink = new TestSink();
    gn.connectInputPort("ip1", windowGenerator.acquireReservoir(String.valueOf(gn.id), 1024));
    gn.connectOutputPort("output", testSink);
    gn.firstWindowMillis = firstWindowMillis;
    gn.windowWidthMillis = windowWidth;
    windowGenerator.activate(null);
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long startTime = System.currentTimeMillis();
    long endTime = 0;
    while (operatorContext.checkpoints.size() < 8 && ((endTime = System.currentTimeMillis()) - startTime) < timeoutMillis) {
        Thread.sleep(sleepTime);
    }
    gn.shutdown();
    t.join();
    windowGenerator.deactivate();
    Assert.assertTrue(!operatorContext.checkpoints.isEmpty());
    for (int index = 0; index < operatorContext.checkpoints.size(); index++) {
        if (operatorContext.checkpoints.get(index) == null) {
            continue;
        }
        Assert.assertEquals(0, operatorContext.checkpoints.get(index).applicationWindowCount);
        Assert.assertEquals(0, operatorContext.checkpoints.get(index).checkpointWindowCount);
    }
}

Example 12 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class NodeTest method testDoubleCheckpointHandling.

@SuppressWarnings("SleepWhileInLoop")
public static void testDoubleCheckpointHandling(ProcessingMode processingMode, boolean trueGenericFalseInput, String path) throws Exception {
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    windowGenerator.setResetWindow(0L);
    windowGenerator.setFirstWindow(0L);
    windowGenerator.setWindowWidth(100);
    windowGenerator.setCheckpointCount(1, 0);
    CheckpointTestOperator checkpointTestOperator;
    if (trueGenericFalseInput) {
        checkpointTestOperator = new CheckpointTestOperator();
    } else {
        checkpointTestOperator = new InputCheckpointTestOperator();
    }
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(com.datatorrent.stram.engine.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.CHECKPOINT_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.PROCESSING_MODE, processingMode);
    dam.put(com.datatorrent.stram.engine.OperatorContext.STORAGE_AGENT, new FSStorageAgent(path, new Configuration()));
    final Node in;
    if (trueGenericFalseInput) {
        in = new GenericNode(checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    } else {
        in = new InputNode((InputCheckpointTestOperator) checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    }
    in.setId(1);
    TestSink testSink = new TestSink();
    String inputPort;
    if (trueGenericFalseInput) {
        inputPort = "ip1";
    } else {
        inputPort = Node.INPUT;
    }
    in.connectInputPort(inputPort, windowGenerator.acquireReservoir(String.valueOf(in.id), 1024));
    in.connectOutputPort("output", testSink);
    in.firstWindowMillis = 0;
    in.windowWidthMillis = 100;
    windowGenerator.activate(null);
    Thread t = new Thread() {

        @Override
        public void run() {
            in.activate();
            in.run();
            in.deactivate();
        }
    };
    t.start();
    long startTime = System.currentTimeMillis();
    long endTime = 0;
    while (checkpointTestOperator.numWindows < 3 && ((endTime = System.currentTimeMillis()) - startTime) < 6000) {
        Thread.sleep(50);
    }
    in.shutdown();
    t.join();
    windowGenerator.deactivate();
    Assert.assertFalse(checkpointTestOperator.checkpointTwice);
    Assert.assertTrue("Timed out", (endTime - startTime) < 5000);
}

Example 13 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class NodeTest method testStatelessOperatorCheckpointing.

@Test
public void testStatelessOperatorCheckpointing() {
    DefaultAttributeMap attributeMap = new DefaultAttributeMap();
    attributeMap.put(OperatorContext.STORAGE_AGENT, new StorageAgentImpl());
    attributeMap.put(OperatorContext.STATELESS, true);
    Node<StatelessOperator> node = new Node<StatelessOperator>(new StatelessOperator(), new com.datatorrent.stram.engine.OperatorContext(0, "operator", attributeMap, null)) {

        @Override
        public void connectInputPort(String port, SweepableReservoir reservoir) {
            //To change body of generated methods, choose Tools | Templates.
            throw new UnsupportedOperationException("Not supported yet.");
        }

        @Override
        public void run() {
            //To change body of generated methods, choose Tools | Templates.
            throw new UnsupportedOperationException("Not supported yet.");
        }
    };
    node.activate();
    synchronized (StorageAgentImpl.calls) {
        StorageAgentImpl.calls.clear();
        node.checkpoint(0);
        Assert.assertEquals("Calls to StorageAgent", 0, StorageAgentImpl.calls.size());
    }
    node.deactivate();
}

Example 14 with DefaultAttributeMap

use of com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap in project apex-core by apache.

the class InlineStreamTest method test.

@Test
@SuppressWarnings("SleepWhileInLoop")
public void test() throws Exception {
    final int totalTupleCount = 5000;
    final PassThroughNode<Object> operator1 = new PassThroughNode<>();
    final GenericNode node1 = new GenericNode(operator1, new OperatorContext(1, "operator1", new DefaultAttributeMap(), null));
    node1.setId(1);
    operator1.setup(node1.context);
    final PassThroughNode<Object> operator2 = new PassThroughNode<>();
    final GenericNode node2 = new GenericNode(operator2, new OperatorContext(2, "operator2", new DefaultAttributeMap(), null));
    node2.setId(2);
    operator2.setup(node2.context);
    StreamContext streamContext = new StreamContext("node1->node2");
    final InlineStream stream = new InlineStream(1024);
    stream.setup(streamContext);
    node1.connectOutputPort("output", stream);
    node2.connectInputPort("input", stream.getReservoir());
    prev = null;
    Sink<Object> sink = new Sink<Object>() {

        @Override
        public void put(Object payload) {
            if (payload instanceof Tuple) {
                return;
            }
            if (prev == null) {
                prev = payload;
            } else {
                if (Integer.valueOf(payload.toString()) - Integer.valueOf(prev.toString()) != 1) {
                    synchronized (InlineStreamTest.this) {
                        InlineStreamTest.this.notify();
                    }
                }
                prev = payload;
            }
            if (Integer.valueOf(prev.toString()) == totalTupleCount - 1) {
                synchronized (InlineStreamTest.this) {
                    InlineStreamTest.this.notify();
                }
            }
        }

        @Override
        public int getCount(boolean reset) {
            return 0;
        }
    };
    node2.connectOutputPort("output", sink);
    AbstractReservoir reservoir1 = AbstractReservoir.newReservoir("input", 1024 * 5);
    node1.connectInputPort("input", reservoir1);
    Map<Integer, Node<?>> activeNodes = new ConcurrentHashMap<>();
    launchNodeThread(node1, activeNodes);
    launchNodeThread(node2, activeNodes);
    stream.activate(streamContext);
    reservoir1.put(StramTestSupport.generateBeginWindowTuple("irrelevant", 0));
    for (int i = 0; i < totalTupleCount; i++) {
        reservoir1.put(i);
    }
    reservoir1.put(StramTestSupport.generateEndWindowTuple("irrelevant", 0));
    synchronized (this) {
        this.wait(200);
    }
    Assert.assertNotNull(prev);
    Assert.assertEquals("processing complete", totalTupleCount, Integer.valueOf(prev.toString()) + 1);
    Assert.assertEquals("active operators", 2, activeNodes.size());
    WaitCondition c = new WaitCondition() {

        @Override
        public boolean isComplete() {
            final SweepableReservoir reservoir = stream.getReservoir();
            logger.debug("stream {} empty {}, size {}", stream, reservoir.isEmpty(), reservoir.size(false));
            return reservoir.isEmpty();
        }
    };
    Assert.assertTrue("operator should finish processing all events within 1 second", StramTestSupport.awaitCompletion(c, 1000));
    stream.deactivate();
    for (Node<?> node : activeNodes.values()) {
        node.shutdown();
    }
    for (int i = 0; i < 10; i++) {
        Thread.sleep(20);
        if (activeNodes.isEmpty()) {
            break;
        }
    }
    stream.teardown();
    operator2.teardown();
    operator1.teardown();
    Assert.assertEquals("active operators", 0, activeNodes.size());
}