use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class AtLeastOnceTest method testInlineOperatorsRecovery.
@Test
public void testInlineOperatorsRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
int maxTuples = 30;
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
asyncFSStorageAgent.setSyncCheckpoint(true);
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
//dag.getAttributes().get(DAG.HEARTBEAT_INTERVAL_MILLIS, 400);
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
cm.setSimulateFailure(true);
dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
// for (Long l: collection) {
// logger.debug(Codec.getStringWindowId(l));
// }
Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class InputOperatorTest method testSomeMethod.
@Test
public void testSomeMethod() throws Exception {
LogicalPlan dag = new LogicalPlan();
String testWorkDir = new File("target").getAbsolutePath();
dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir, null));
EvenOddIntegerGeneratorInputOperator generator = dag.addOperator("NumberGenerator", EvenOddIntegerGeneratorInputOperator.class);
final CollectorModule<Number> collector = dag.addOperator("NumberCollector", new CollectorModule<Number>());
dag.addStream("EvenIntegers", generator.even, collector.even).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("OddIntegers", generator.odd, collector.odd).setLocality(Locality.CONTAINER_LOCAL);
final StramLocalCluster lc = new StramLocalCluster(dag);
lc.setHeartbeatMonitoringEnabled(false);
lc.runAsync();
WaitCondition c = new WaitCondition() {
@Override
public boolean isComplete() {
return tupleCount.get() > 2;
}
};
StramTestSupport.awaitCompletion(c, 2000);
lc.shutdown();
Assert.assertEquals("Collections size", 2, collections.size());
Assert.assertFalse("Zero tuple count", collections.get(collector.even.id).isEmpty() && collections.get(collector.odd.id).isEmpty());
Assert.assertTrue("Tuple count", collections.get(collector.even.id).size() - collections.get(collector.odd.id).size() <= 1);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class ProcessingModeTests method testLinearInlineOperatorsRecovery.
public void testLinearInlineOperatorsRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
CollectorOperator.duplicates.clear();
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
cm.setSimulateFailure(true);
dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);
dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class SliderTest method test.
private void test(int applicationWindowCount, int slideByWindowCount) throws Exception {
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/sliderTest").getAbsolutePath();
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 100);
Input input = dag.addOperator("Input", new Input());
Sum sum = dag.addOperator("Sum", new Sum());
dag.setOperatorAttribute(sum, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindowCount);
dag.setOperatorAttribute(sum, OperatorContext.SLIDE_BY_WINDOW_COUNT, slideByWindowCount);
Validator validate = dag.addOperator("validator", new Validator());
Validator.numbersValidated = 0;
validate.numberOfIntegers = applicationWindowCount;
validate.slideByNumbers = slideByWindowCount;
dag.addStream("input-sum", input.defaultOutputPort, sum.inputPort);
dag.addStream("sum-validator", sum.outputPort, validate.validate);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
long startTms = System.currentTimeMillis();
while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
if (validate.numbersValidated > 5) {
break;
}
Thread.sleep(100);
}
lc.shutdown();
Assert.assertTrue("numbers validated more than zero ", validate.numbersValidated > 0);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class StatsTest method testPortStatsPropagation.
/**
* Verify buffer server bytes and tuple count.
*
* @throws Exception
*/
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testPortStatsPropagation() throws Exception {
int tupleCount = 10;
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target").getAbsolutePath();
dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
TestInputStatsListener testInputStatsListener = new TestInputStatsListener();
dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testInputStatsListener }));
testOper.setMaxTuples(tupleCount);
testOper.setEmitInterval(0);
TestCollector collector = dag.addOperator("Collector", new TestCollector());
TestCollectorStatsListener testCollectorStatsListener = new TestCollectorStatsListener();
dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { testCollectorStatsListener }));
dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(null);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
Assert.assertFalse("input operator stats", testInputStatsListener.inputOperatorStats.isEmpty());
Assert.assertFalse("collector operator stats", testCollectorStatsListener.collectorOperatorStats.isEmpty());
try {
int outputPortTupleCount = 0;
long outputPortBufferServerBytes = 0L;
for (Iterator<OperatorStats> it = testInputStatsListener.inputOperatorStats.iterator(); it.hasNext(); ) {
OperatorStats operatorStats = it.next();
for (PortStats outputPortStats : operatorStats.outputPorts) {
outputPortTupleCount += outputPortStats.tupleCount;
outputPortBufferServerBytes += outputPortStats.bufferServerBytes;
}
}
int inputPortTupleCount = 0;
long inputPortBufferServerBytes = 0L;
for (Iterator<OperatorStats> it = testCollectorStatsListener.collectorOperatorStats.iterator(); it.hasNext(); ) {
OperatorStats operatorStats = it.next();
for (PortStats inputPortStats : operatorStats.inputPorts) {
inputPortTupleCount += inputPortStats.tupleCount;
inputPortBufferServerBytes += inputPortStats.bufferServerBytes;
}
}
Assert.assertEquals("Tuple Count emitted", tupleCount, outputPortTupleCount);
Assert.assertTrue("Buffer server bytes", inputPortBufferServerBytes > 0);
Assert.assertEquals("Tuple Count processed", tupleCount, inputPortTupleCount);
Assert.assertTrue("Buffer server bytes", outputPortBufferServerBytes > 0);
} finally {
lc.shutdown();
}
}
Aggregations