use of com.datatorrent.stram.support.ManualScheduledExecutorService in project apex-core by apache.
the class WindowGeneratorTest method test2ndResetWindow.
@Test
public void test2ndResetWindow() throws InterruptedException {
logger.info("Testing 2nd Reset Window");
ManualScheduledExecutorService msse = new ManualScheduledExecutorService(1);
WindowGenerator generator = new WindowGenerator(msse, (WindowGenerator.MAX_WINDOW_ID << 1) + 1024);
generator.setFirstWindow(0L);
generator.setResetWindow(0L);
generator.setWindowWidth(1);
SweepableReservoir reservoir = generator.acquireReservoir(Node.OUTPUT, (WindowGenerator.MAX_WINDOW_ID << 1) + 1024);
final AtomicBoolean loggingEnabled = new AtomicBoolean(true);
reservoir.setSink(new Sink<Object>() {
@Override
public void put(Object payload) {
if (loggingEnabled.get()) {
logger.debug(payload.toString());
}
}
@Override
public int getCount(boolean reset) {
return 0;
}
});
generator.activate(null);
msse.tick(1);
/* reset window and begin window */
msse.tick(1);
/* end window and begin window */
loggingEnabled.set(false);
for (int i = 0; i < WindowGenerator.MAX_WINDOW_ID - 2; i++) {
msse.tick(1);
/* end window and begin window */
}
loggingEnabled.set(true);
msse.tick(1);
/* end window, reset window, begin window */
final AtomicInteger beginWindowCount = new AtomicInteger(0);
final AtomicInteger endWindowCount = new AtomicInteger(0);
final AtomicInteger resetWindowCount = new AtomicInteger(0);
Tuple t;
reservoir.sweep();
while ((t = reservoir.sweep()) != null) {
reservoir.remove();
switch(t.getType()) {
case BEGIN_WINDOW:
beginWindowCount.incrementAndGet();
break;
case END_WINDOW:
endWindowCount.incrementAndGet();
break;
case RESET_WINDOW:
resetWindowCount.incrementAndGet();
break;
default:
break;
}
}
Assert.assertEquals("begin windows", WindowGenerator.MAX_WINDOW_ID + 1 + 1, beginWindowCount.get());
Assert.assertEquals("end windows", WindowGenerator.MAX_WINDOW_ID + 1, endWindowCount.get());
Assert.assertEquals("reset windows", 2, resetWindowCount.get());
}
use of com.datatorrent.stram.support.ManualScheduledExecutorService in project apex-core by apache.
the class WindowGeneratorTest method testResetWindow.
/**
* Test of resetWindow functionality of WindowGenerator.
*/
@Test
public void testResetWindow() {
ManualScheduledExecutorService msse = new ManualScheduledExecutorService(1);
msse.setCurrentTimeMillis(0x0afebabe * 1000L);
WindowGenerator generator = new WindowGenerator(msse, WindowGenerator.MAX_WINDOW_ID + 1024);
final long currentTIme = msse.getCurrentTimeMillis();
final int windowWidth = 0x1234abcd;
generator.setFirstWindow(currentTIme);
generator.setResetWindow(currentTIme);
generator.setWindowWidth(windowWidth);
SweepableReservoir reservoir = generator.acquireReservoir(Node.OUTPUT, 1024);
reservoir.setSink(new Sink<Object>() {
boolean firsttime = true;
@Override
public int getCount(boolean reset) {
return 0;
}
@Override
public void put(Object payload) {
assert (false);
if (firsttime) {
assert (payload instanceof ResetWindowTuple);
firsttime = false;
} else {
assert (payload instanceof Tuple);
}
}
});
generator.activate(null);
msse.tick(1);
Assert.assertNull(reservoir.sweep());
ResetWindowTuple rwt = (ResetWindowTuple) reservoir.sweep();
reservoir.remove();
assert (rwt.getWindowId() == 0x0afebabe00000000L);
assert (rwt.getBaseSeconds() * 1000L == currentTIme);
assert (rwt.getIntervalMillis() == windowWidth);
Tuple t = reservoir.sweep();
reservoir.remove();
assert (t.getType() == MessageType.BEGIN_WINDOW);
assert (t.getWindowId() == 0x0afebabe00000000L);
assert (reservoir.sweep() == null);
}
use of com.datatorrent.stram.support.ManualScheduledExecutorService in project apex-core by apache.
the class StramLocalClusterTest method testRecovery.
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testRecovery() throws Exception {
AsyncFSStorageAgent agent = new AsyncFSStorageAgent(testMeta.getPath(), null);
agent.setSyncCheckpoint(true);
dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
TestGeneratorInputOperator node1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
// data will be added externally from test
node1.setMaxTuples(0);
GenericTestOperator node2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("o1o2", node1.outport, node2.inport1);
dag.validate();
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
final ManualScheduledExecutorService wclock = new ManualScheduledExecutorService(1);
MockComponentFactory mcf = new MockComponentFactory() {
@Override
public WindowGenerator setupWindowGenerator() {
WindowGenerator wingen = StramTestSupport.setupWindowGenerator(wclock);
wingen.setCheckpointCount(2, 0);
return wingen;
}
};
StramLocalCluster localCluster = new StramLocalCluster(dag, mcf);
localCluster.setPerContainerBufferServer(true);
// driven by test
localCluster.setHeartbeatMonitoringEnabled(false);
localCluster.runAsync();
PTOperator ptNode1 = localCluster.findByLogicalNode(dag.getMeta(node1));
PTOperator ptNode2 = localCluster.findByLogicalNode(dag.getMeta(node2));
LocalStreamingContainer c0 = StramTestSupport.waitForActivation(localCluster, ptNode1);
Map<Integer, Node<?>> nodeMap = c0.getNodes();
Assert.assertEquals("number operators", 1, nodeMap.size());
TestGeneratorInputOperator n1 = (TestGeneratorInputOperator) nodeMap.get(ptNode1.getId()).getOperator();
Assert.assertNotNull(n1);
LocalStreamingContainer c2 = StramTestSupport.waitForActivation(localCluster, ptNode2);
Map<Integer, Node<?>> c2NodeMap = c2.getNodes();
Assert.assertEquals("number operators downstream", 1, c2NodeMap.size());
GenericTestOperator n2 = (GenericTestOperator) c2NodeMap.get(localCluster.findByLogicalNode(dag.getMeta(node2)).getId()).getOperator();
Assert.assertNotNull(n2);
// input data
String tuple1 = "tuple1";
n1.addTuple(tuple1);
OperatorContext n1Context = c0.getNodeContext(ptNode1.getId());
Assert.assertEquals("initial window id", -1, n1Context.getLastProcessedWindowId());
// checkpoint window
wclock.tick(1);
wclock.tick(1);
Assert.assertEquals("current window", 2, wclock.getCurrentTimeMillis());
OperatorContext o2Context = c2.getNodeContext(ptNode2.getId());
Assert.assertNotNull("context ", o2Context);
StramTestSupport.waitForWindowComplete(o2Context, 1);
Assert.assertEquals("o2 received ", tuple1, n2.inport1Tuple);
wclock.tick(1);
Assert.assertEquals("current window", 3, wclock.getCurrentTimeMillis());
// checkpoint between window 1 and 2
StramTestSupport.waitForWindowComplete(o2Context, 2);
// propagate checkpoints to master
c0.triggerHeartbeat();
// wait for heartbeat cycle to complete
c0.waitForHeartbeat(5000);
Assert.assertEquals("checkpoint " + ptNode1, 1, ptNode1.getRecentCheckpoint().windowId);
c2.triggerHeartbeat();
//Thread.yield();
// yield without using yield for heartbeat cycle
Thread.sleep(1);
c2.waitForHeartbeat(5000);
Assert.assertEquals("checkpoint " + ptNode2, 1, ptNode2.getRecentCheckpoint().windowId);
Assert.assertEquals("checkpoints " + ptNode1, Arrays.asList(new Checkpoint[] { new Checkpoint(1L, 0, 0) }), ptNode1.checkpoints);
Assert.assertEquals("checkpoints " + ptNode2, Arrays.asList(new Checkpoint[] { new Checkpoint(1L, 0, 0) }), ptNode2.checkpoints);
//
// simulate container failure (operator o1)
//
localCluster.failContainer(c0);
// replacement container starts empty
// operators will deploy after downstream operator was removed
LocalStreamingContainer c0Replaced = StramTestSupport.waitForActivation(localCluster, ptNode1);
c0Replaced.triggerHeartbeat();
// next heartbeat after setup
c0Replaced.waitForHeartbeat(5000);
Assert.assertNotSame("old container", c0, c0Replaced);
Assert.assertNotSame("old container", c0.getContainerId(), c0Replaced.getContainerId());
// verify change in downstream container
LOG.debug("triggering c2 heartbeat processing");
StreamingContainerAgent c2Agent = localCluster.getContainerAgent(c2);
// wait for downstream re-deploy to complete
long startTms = System.currentTimeMillis();
while (c2Agent.hasPendingWork() && StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
Thread.sleep(200);
c2.triggerHeartbeat();
LOG.debug("Waiting for {} to complete pending work.", c2.getContainerId());
}
Assert.assertEquals(c2.getContainerId() + " operators after redeploy " + c2.getNodes(), 1, c2.getNodes().size());
// verify downstream operator re-deployed in existing container
Assert.assertEquals("active " + ptNode2, c2, StramTestSupport.waitForActivation(localCluster, ptNode2));
GenericTestOperator o2Recovered = (GenericTestOperator) c2NodeMap.get(localCluster.findByLogicalNode(dag.getMeta(node2)).getId()).getOperator();
Assert.assertNotNull("redeployed " + ptNode2, o2Recovered);
Assert.assertNotSame("new instance " + ptNode2, n2, o2Recovered);
Assert.assertEquals("restored state " + ptNode2, tuple1, o2Recovered.inport1Tuple);
TestGeneratorInputOperator o1Recovered = (TestGeneratorInputOperator) c0Replaced.getNodes().get(ptNode1.getId()).getOperator();
Assert.assertNotNull(o1Recovered);
OperatorContext o1RecoveredContext = c0Replaced.getNodeContext(ptNode1.getId());
Assert.assertNotNull("active " + ptNode1, o1RecoveredContext);
wclock.tick(1);
Assert.assertEquals("current window", 4, wclock.getCurrentTimeMillis());
// refresh context after operator re-deploy
o2Context = c2.getNodeContext(ptNode2.getId());
Assert.assertNotNull("active " + ptNode2, o2Context);
StramTestSupport.waitForWindowComplete(o1RecoveredContext, 3);
StramTestSupport.waitForWindowComplete(o2Context, 3);
// checkpoint window
wclock.tick(1);
Assert.assertEquals("current window", 5, wclock.getCurrentTimeMillis());
String tuple2 = "tuple2";
o1Recovered.addTuple(tuple2);
StramTestSupport.waitForWindowComplete(o1RecoveredContext, 4);
StramTestSupport.waitForWindowComplete(o2Context, 4);
// check data flow after recovery
Assert.assertEquals("retrieved tuple (after recovery) " + ptNode2, tuple2, o2Recovered.inport1Tuple);
// propagate checkpoints to master
c0Replaced.triggerHeartbeat();
c0Replaced.waitForHeartbeat(5000);
c2.triggerHeartbeat();
c2.waitForHeartbeat(5000);
// purge checkpoints
// checkpoint purging
localCluster.dnmgr.monitorHeartbeat(false);
Assert.assertEquals("checkpoints " + ptNode1, Arrays.asList(new Checkpoint[] { new Checkpoint(3L, 0, 0) }), ptNode1.checkpoints);
Assert.assertEquals("checkpoints " + ptNode2, Arrays.asList(new Checkpoint[] { new Checkpoint(3L, 0, 0) }), ptNode2.checkpoints);
localCluster.shutdown();
}
Aggregations