use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class StreamingContainerTest method testOiOCommitted.
@Test
public void testOiOCommitted() throws IOException, ClassNotFoundException {
LogicalPlan lp = new LogicalPlan();
String workingDir = new File("target/testCommitted").getAbsolutePath();
lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
String op1Name = "CommitAwareOperatorTestOioCommit1";
String op2Name = "CommitAwareOperatorTestOioCommit2";
CommitAwareOperator operator1 = lp.addOperator(op1Name, new CommitAwareOperator());
CommitAwareOperator operator2 = lp.addOperator(op2Name, new CommitAwareOperator());
lp.addStream("local", operator1.output, operator2.input).setLocality(Locality.THREAD_LOCAL);
StramLocalCluster lc = new StramLocalCluster(lp);
lc.run(5000);
/* this is not foolproof but some insurance is better than nothing */
Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op1Name));
Assert.assertTrue("No Committed Windows", committedWindowIds.contains(op2Name));
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class StreamingContainerTest method testCommitted.
@Test
public void testCommitted() throws IOException, ClassNotFoundException {
LogicalPlan lp = new LogicalPlan();
String workingDir = new File("target/testCommitted").getAbsolutePath();
lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
lp.setAttribute(DAGContext.CHECKPOINT_WINDOW_COUNT, 1);
String opName = "CommitAwareOperatorTestCommit";
lp.addOperator(opName, new CommitAwareOperator());
StramLocalCluster lc = new StramLocalCluster(lp);
lc.run(5000);
/* this is not foolproof but some insurance is better than nothing */
Assert.assertTrue("No Committed Windows", committedWindowIds.contains(opName));
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class WindowGeneratorTest method testOutofSequenceError.
@Test
public void testOutofSequenceError() throws Exception {
logger.info("Testing Out of Sequence Error");
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/testOutofSequenceError").getAbsolutePath();
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
RandomNumberGenerator rng = dag.addOperator("random", new RandomNumberGenerator());
MyLogger ml = dag.addOperator("logger", new MyLogger());
dag.addStream("stream", rng.output, ml.input);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run(10000);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class OiOStreamTest method validateOiOiOImplementation.
@Test
public void validateOiOiOImplementation() throws Exception {
LogicalPlan lp = new LogicalPlan();
lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
ThreadIdValidatingInputOperator inputOperator = lp.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
ThreadIdValidatingGenericIntermediateOperator intermediateOperator = lp.addOperator("intermediateOperator", new ThreadIdValidatingGenericIntermediateOperator());
ThreadIdValidatingOutputOperator outputOperator = lp.addOperator("outputOperator", new ThreadIdValidatingOutputOperator());
StreamMeta stream1 = lp.addStream("OiO1", inputOperator.output, intermediateOperator.input);
StreamMeta stream2 = lp.addStream("OiO2", intermediateOperator.output, outputOperator.input);
StramLocalCluster slc;
/* The first test makes sure that when they are not ThreadLocal they use different threads */
ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
lp.validate();
slc = new StramLocalCluster(lp);
slc.run();
Assert.assertFalse("Thread Id 1", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingGenericIntermediateOperator.threadId);
Assert.assertFalse("Thread Id 2", ThreadIdValidatingGenericIntermediateOperator.threadId == ThreadIdValidatingOutputOperator.threadId);
/* This test makes sure that since they are ThreadLocal, they indeed share a thread */
ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
stream1.setLocality(Locality.THREAD_LOCAL);
stream2.setLocality(Locality.THREAD_LOCAL);
lp.validate();
slc = new StramLocalCluster(lp);
slc.run();
Assert.assertEquals("Thread Id 3", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericIntermediateOperator.threadId);
Assert.assertEquals("Thread Id 4", ThreadIdValidatingGenericIntermediateOperator.threadId, ThreadIdValidatingOutputOperator.threadId);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class OiOStreamTest method validateOiOiOTreeImplementation.
@Test
public void validateOiOiOTreeImplementation() throws Exception {
LogicalPlan lp = new LogicalPlan();
lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
ThreadIdValidatingInputOperator inputOperator1 = lp.addOperator("inputOperator1", new ThreadIdValidatingInputOperator());
ThreadIdValidatingGenericIntermediateOperator intermediateOperatorfromInputOper1 = lp.addOperator("intermediateOperatorfromInputOper1", new ThreadIdValidatingGenericIntermediateOperator());
ThreadIdValidatingGenericIntermediateOperator intermediateOperatorfromInterOper11 = lp.addOperator("intermediateOperatorfromInterOper11", new ThreadIdValidatingGenericIntermediateOperator());
ThreadIdValidatingGenericIntermediateOperator intermediateOperatorfromInterOper12 = lp.addOperator("intermediateOperatorfromInterOper12", new ThreadIdValidatingGenericIntermediateOperator());
ThreadIdValidatingOutputOperator outputOperatorFromInputOper = lp.addOperator("outputOperatorFromInputOper", new ThreadIdValidatingOutputOperator());
ThreadIdValidatingOutputOperator outputOperatorFromInterOper11 = lp.addOperator("outputOperatorFromInterOper11", new ThreadIdValidatingOutputOperator());
ThreadIdValidatingOutputOperator outputOperatorFromInterOper21 = lp.addOperator("outputOperatorFromInterOper21", new ThreadIdValidatingOutputOperator());
ThreadIdValidatingOutputOperator outputOperatorFromInterOper22 = lp.addOperator("outputOperatorFromInterOper22", new ThreadIdValidatingOutputOperator());
StreamMeta stream1 = lp.addStream("OiO1", inputOperator1.output, outputOperatorFromInputOper.input, intermediateOperatorfromInputOper1.input);
StreamMeta stream2 = lp.addStream("OiO2", intermediateOperatorfromInputOper1.output, intermediateOperatorfromInterOper11.input, intermediateOperatorfromInterOper12.input);
StreamMeta stream3 = lp.addStream("OiO3", intermediateOperatorfromInterOper11.output, outputOperatorFromInterOper11.input);
lp.addStream("nonOiO1", intermediateOperatorfromInterOper12.output, outputOperatorFromInterOper21.input, outputOperatorFromInterOper22.input);
StramLocalCluster slc;
/*
* This test makes sure that since no operators in dag tree are ThreadLocal, they dont share threads
*/
ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
ThreadIdValidatingOutputOperator.threadList.clear();
lp.validate();
slc = new StramLocalCluster(lp);
slc.run();
Assert.assertEquals("nonOIO: Number of threads ThreadIdValidatingGenericIntermediateOperator", 3, ThreadIdValidatingGenericIntermediateOperator.threadList.size());
Assert.assertEquals("nonOIO: Number of unique threads ThreadIdValidatingGenericIntermediateOperator", 3, (new HashSet<>(ThreadIdValidatingGenericIntermediateOperator.threadList)).size());
Assert.assertEquals("nonOIO: Number of threads ThreadIdValidatingOutputOperator", 4, ThreadIdValidatingOutputOperator.threadList.size());
Assert.assertEquals("nonOIO: Number of unique threads ThreadIdValidatingOutputOperator", 4, (new HashSet<>(ThreadIdValidatingOutputOperator.threadList)).size());
Assert.assertFalse("nonOIO:: inputOperator1 : ThreadIdValidatingOutputOperator", ThreadIdValidatingOutputOperator.threadList.contains(ThreadIdValidatingInputOperator.threadId));
Assert.assertFalse("nonOIO:: inputOperator1 : ThreadIdValidatingGenericIntermediateOperator", ThreadIdValidatingGenericIntermediateOperator.threadList.contains(ThreadIdValidatingInputOperator.threadId));
/*
* This test makes sure that since some operators in the dag tree are ThreadLocal, they indeed share a thread
*/
ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
ThreadIdValidatingOutputOperator.threadList.clear();
stream1.setLocality(Locality.THREAD_LOCAL);
stream2.setLocality(Locality.THREAD_LOCAL);
stream3.setLocality(Locality.THREAD_LOCAL);
lp.validate();
slc = new StramLocalCluster(lp);
slc.run();
Assert.assertEquals("OIO: Number of threads ThreadIdValidatingGenericIntermediateOperator", 3, ThreadIdValidatingGenericIntermediateOperator.threadList.size());
Assert.assertEquals("OIO: Number of unique threads ThreadIdValidatingGenericIntermediateOperator", 1, (new HashSet<>(ThreadIdValidatingGenericIntermediateOperator.threadList)).size());
Assert.assertEquals("OIO: Number of threads ThreadIdValidatingOutputOperator", 4, ThreadIdValidatingOutputOperator.threadList.size());
Assert.assertEquals("OIO: Number of unique threads ThreadIdValidatingOutputOperator", 3, (new HashSet<>(ThreadIdValidatingOutputOperator.threadList)).size());
Assert.assertTrue("OIO:: inputOperator1 : ThreadIdValidatingOutputOperator", ThreadIdValidatingOutputOperator.threadList.contains(ThreadIdValidatingInputOperator.threadId));
Assert.assertTrue("OIO:: inputOperator1 : ThreadIdValidatingGenericIntermediateOperator", ThreadIdValidatingGenericIntermediateOperator.threadList.contains(ThreadIdValidatingInputOperator.threadId));
}
Aggregations