Examples with ScheduledThreadPoolExecutor com.datatorrent.common.util.ScheduledThreadPoolExecutor used on opensource projects

Example 1 with ScheduledThreadPoolExecutor

use of com.datatorrent.common.util.ScheduledThreadPoolExecutor in project apex-core by apache.

the class StreamingContainer method setupWindowGenerator.

/**
 * Create the window generator for the given start window id.
 * This is a hook for tests to control the window generation.
 *
 * @param finishedWindowId
 * @return WindowGenerator
 */
protected WindowGenerator setupWindowGenerator(long finishedWindowId) {
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    /**
     * let's make sure that we send the same window Ids with the same reset windows.
     */
    windowGenerator.setResetWindow(firstWindowMillis);
    long millisAtFirstWindow = WindowGenerator.getNextWindowMillis(finishedWindowId, firstWindowMillis, windowWidthMillis);
    windowGenerator.setFirstWindow(millisAtFirstWindow);
    windowGenerator.setWindowWidth(windowWidthMillis);
    long windowCount = WindowGenerator.getWindowCount(millisAtFirstWindow, firstWindowMillis, windowWidthMillis);
    windowGenerator.setCheckpointCount(checkpointWindowCount, (int) (windowCount % checkpointWindowCount));
    return windowGenerator;
}

Example 2 with ScheduledThreadPoolExecutor

use of com.datatorrent.common.util.ScheduledThreadPoolExecutor in project apex-core by apache.

the class WindowGeneratorTest method testWindowGen.

@Test
public void testWindowGen() throws Exception {
    final AtomicLong currentWindow = new AtomicLong();
    final AtomicInteger beginWindowCount = new AtomicInteger();
    final AtomicInteger endWindowCount = new AtomicInteger();
    final AtomicLong windowXor = new AtomicLong();
    Sink<Object> s = new Sink<Object>() {

        @Override
        public int getCount(boolean reset) {
            return 0;
        }

        @Override
        public void put(Object payload) {
            logger.debug("unexpected payload {}", payload);
        }
    };
    ScheduledThreadPoolExecutor stpe = new ScheduledThreadPoolExecutor(1, "WindowGenerator");
    int windowWidth = 200;
    long firstWindowMillis = stpe.getCurrentTimeMillis();
    firstWindowMillis -= firstWindowMillis % 1000L;
    WindowGenerator wg = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), WindowGenerator.MAX_WINDOW_ID + 1024);
    wg.setResetWindow(firstWindowMillis);
    wg.setFirstWindow(firstWindowMillis);
    wg.setWindowWidth(windowWidth);
    SweepableReservoir reservoir = wg.acquireReservoir("GeneratorTester", windowWidth);
    reservoir.setSink(s);
    wg.activate(null);
    Thread.sleep(200);
    wg.deactivate();
    reservoir.sweep();
    /* just transfer over all the control tuples */
    Tuple t;
    while ((t = reservoir.sweep()) != null) {
        reservoir.remove();
        long windowId = t.getWindowId();
        switch(t.getType()) {
            case BEGIN_WINDOW:
                currentWindow.set(windowId);
                beginWindowCount.incrementAndGet();
                windowXor.set(windowXor.get() ^ windowId);
                break;
            case END_WINDOW:
                endWindowCount.incrementAndGet();
                windowXor.set(windowXor.get() ^ windowId);
                break;
            case RESET_WINDOW:
                break;
            default:
                currentWindow.set(0);
                break;
        }
    }
    long lastWindowMillis = System.currentTimeMillis();
    Assert.assertEquals("only last window open", currentWindow.get(), windowXor.get());
    long expectedCnt = (lastWindowMillis - firstWindowMillis) / windowWidth;
    Assert.assertTrue("Minimum begin window count", expectedCnt + 1 <= beginWindowCount.get());
    Assert.assertEquals("end window count", beginWindowCount.get() - 1, endWindowCount.get());
}

Example 3 with ScheduledThreadPoolExecutor

use of com.datatorrent.common.util.ScheduledThreadPoolExecutor in project apex-core by apache.

the class GenericNodeTest method testCheckpointApplicationWindowCount.

private void testCheckpointApplicationWindowCount(ProcessingMode processingMode) throws Exception {
    final long timeoutMillis = 10000L;
    final long sleepTime = 25L;
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    long resetWindow = 0L;
    long firstWindowMillis = 1448909287863L;
    int windowWidth = 100;
    windowGenerator.setResetWindow(resetWindow);
    windowGenerator.setFirstWindow(firstWindowMillis);
    windowGenerator.setWindowWidth(windowWidth);
    windowGenerator.setCheckpointCount(1, 0);
    GenericOperator go = new GenericOperator();
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(OperatorContext.APPLICATION_WINDOW_COUNT, 5);
    dam.put(OperatorContext.CHECKPOINT_WINDOW_COUNT, 5);
    dam.put(OperatorContext.PROCESSING_MODE, processingMode);
    DelayAsyncFSStorageAgent storageAgent = new DelayAsyncFSStorageAgent(testMeta.getDir(), new Configuration());
    storageAgent.setDelayMS(200L);
    dam.put(OperatorContext.STORAGE_AGENT, storageAgent);
    TestStatsOperatorContext operatorContext = new TestStatsOperatorContext(0, "operator", dam, null);
    final GenericNode gn = new GenericNode(go, operatorContext);
    gn.setId(1);
    TestSink testSink = new TestSink();
    gn.connectInputPort("ip1", windowGenerator.acquireReservoir(String.valueOf(gn.id), 1024));
    gn.connectOutputPort("output", testSink);
    gn.firstWindowMillis = firstWindowMillis;
    gn.windowWidthMillis = windowWidth;
    windowGenerator.activate(null);
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.activate();
            gn.run();
            gn.deactivate();
        }
    };
    t.start();
    long startTime = System.currentTimeMillis();
    long endTime = 0;
    while (operatorContext.checkpoints.size() < 8 && ((endTime = System.currentTimeMillis()) - startTime) < timeoutMillis) {
        Thread.sleep(sleepTime);
    }
    gn.shutdown();
    t.join();
    windowGenerator.deactivate();
    Assert.assertTrue(!operatorContext.checkpoints.isEmpty());
    for (int index = 0; index < operatorContext.checkpoints.size(); index++) {
        if (operatorContext.checkpoints.get(index) == null) {
            continue;
        }
        Assert.assertEquals(0, operatorContext.checkpoints.get(index).applicationWindowCount);
        Assert.assertEquals(0, operatorContext.checkpoints.get(index).checkpointWindowCount);
    }
}

Example 4 with ScheduledThreadPoolExecutor

use of com.datatorrent.common.util.ScheduledThreadPoolExecutor in project apex-core by apache.

the class GenericNodeTest method testCheckpointDistance.

private void testCheckpointDistance(int dagCheckPoint, int opCheckPoint) throws InterruptedException {
    int windowWidth = 50;
    long sleeptime = 25L;
    int maxWindows = 60;
    // Adding some extra time for the windows to finish
    long maxSleep = windowWidth * maxWindows + 5000;
    ScheduledExecutorService executorService = new ScheduledThreadPoolExecutor(1, "default");
    final WindowGenerator windowGenerator = new WindowGenerator(executorService, 1024);
    windowGenerator.setWindowWidth(windowWidth);
    windowGenerator.setFirstWindow(executorService.getCurrentTimeMillis());
    windowGenerator.setCheckpointCount(dagCheckPoint, 0);
    // GenericOperator go = new GenericOperator();
    CheckpointDistanceOperator go = new CheckpointDistanceOperator();
    go.maxWindows = maxWindows;
    List<Integer> checkpoints = new ArrayList<>();
    int window = 0;
    while (window < maxWindows) {
        window = (int) Math.ceil((double) (window + 1) / dagCheckPoint) * dagCheckPoint;
        window = (int) Math.ceil((double) window / opCheckPoint) * opCheckPoint;
        checkpoints.add(window);
    }
    final StreamContext stcontext = new StreamContext("s1");
    DefaultAttributeMap attrMap = new DefaultAttributeMap();
    attrMap.put(Context.DAGContext.CHECKPOINT_WINDOW_COUNT, dagCheckPoint);
    attrMap.put(Context.OperatorContext.CHECKPOINT_WINDOW_COUNT, opCheckPoint);
    final OperatorContext context = new com.datatorrent.stram.engine.OperatorContext(0, "operator", attrMap, null);
    final GenericNode gn = new GenericNode(go, context);
    gn.setId(1);
    // DefaultReservoir reservoir1 = new DefaultReservoir("ip1Res", 1024);
    // DefaultReservoir reservoir2 = new DefaultReservoir("ip2Res", 1024);
    // gn.connectInputPort("ip1", reservoir1);
    // gn.connectInputPort("ip2", reservoir2);
    gn.connectInputPort("ip1", windowGenerator.acquireReservoir("ip1", 1024));
    gn.connectInputPort("ip2", windowGenerator.acquireReservoir("ip2", 1024));
    gn.connectOutputPort("op", Sink.BLACKHOLE);
    final AtomicBoolean ab = new AtomicBoolean(false);
    Thread t = new Thread() {

        @Override
        public void run() {
            gn.setup(context);
            windowGenerator.activate(stcontext);
            gn.activate();
            ab.set(true);
            gn.run();
            windowGenerator.deactivate();
            gn.deactivate();
            gn.teardown();
        }
    };
    t.start();
    long interval = 0;
    do {
        Thread.sleep(sleeptime);
        interval += sleeptime;
    } while ((go.numWindows < maxWindows) && (interval < maxSleep));
    Assert.assertEquals("Number distances", maxWindows, go.numWindows);
    int chkindex = 0;
    int nextCheckpoint = checkpoints.get(chkindex++);
    for (int i = 0; i < maxWindows; ++i) {
        if ((i + 1) > nextCheckpoint) {
            nextCheckpoint = checkpoints.get(chkindex++);
        }
        Assert.assertEquals("Windows from checkpoint for " + i, nextCheckpoint - i, (int) go.distances.get(i));
    }
    gn.shutdown();
    t.join();
}

Example 5 with ScheduledThreadPoolExecutor

use of com.datatorrent.common.util.ScheduledThreadPoolExecutor in project apex-core by apache.

the class NodeTest method testDoubleCheckpointHandling.

@SuppressWarnings("SleepWhileInLoop")
public static void testDoubleCheckpointHandling(ProcessingMode processingMode, boolean trueGenericFalseInput, String path) throws Exception {
    WindowGenerator windowGenerator = new WindowGenerator(new ScheduledThreadPoolExecutor(1, "WindowGenerator"), 1024);
    windowGenerator.setResetWindow(0L);
    windowGenerator.setFirstWindow(0L);
    windowGenerator.setWindowWidth(100);
    windowGenerator.setCheckpointCount(1, 0);
    CheckpointTestOperator checkpointTestOperator;
    if (trueGenericFalseInput) {
        checkpointTestOperator = new CheckpointTestOperator();
    } else {
        checkpointTestOperator = new InputCheckpointTestOperator();
    }
    DefaultAttributeMap dam = new DefaultAttributeMap();
    dam.put(com.datatorrent.stram.engine.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.CHECKPOINT_WINDOW_COUNT, 2);
    dam.put(com.datatorrent.stram.engine.OperatorContext.PROCESSING_MODE, processingMode);
    dam.put(com.datatorrent.stram.engine.OperatorContext.STORAGE_AGENT, new FSStorageAgent(path, new Configuration()));
    final Node in;
    if (trueGenericFalseInput) {
        in = new GenericNode(checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    } else {
        in = new InputNode((InputCheckpointTestOperator) checkpointTestOperator, new com.datatorrent.stram.engine.OperatorContext(0, "operator", dam, null));
    }
    in.setId(1);
    TestSink testSink = new TestSink();
    String inputPort;
    if (trueGenericFalseInput) {
        inputPort = "ip1";
    } else {
        inputPort = Node.INPUT;
    }
    in.connectInputPort(inputPort, windowGenerator.acquireReservoir(String.valueOf(in.id), 1024));
    in.connectOutputPort("output", testSink);
    in.firstWindowMillis = 0;
    in.windowWidthMillis = 100;
    windowGenerator.activate(null);
    Thread t = new Thread() {

        @Override
        public void run() {
            in.activate();
            in.run();
            in.deactivate();
        }
    };
    t.start();
    long startTime = System.currentTimeMillis();
    long endTime = 0;
    while (checkpointTestOperator.numWindows < 3 && ((endTime = System.currentTimeMillis()) - startTime) < 6000) {
        Thread.sleep(50);
    }
    in.shutdown();
    t.join();
    windowGenerator.deactivate();
    Assert.assertFalse(checkpointTestOperator.checkpointTwice);
    Assert.assertTrue("Timed out", (endTime - startTime) < 5000);
}