use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class ProcessingModeTests method testLinearInputOperatorRecovery.
public void testLinearInputOperatorRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
CollectorOperator.duplicates.clear();
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
dag.getMeta(rip).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
if (Operator.ProcessingMode.EXACTLY_ONCE.equals(processingMode)) {
dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
}
dag.addStream("connection", rip.output, cm.input);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class ProcessingModeTests method testLinearOperatorRecovery.
public void testLinearOperatorRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
CollectorOperator.duplicates.clear();
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
cm.setSimulateFailure(true);
dag.getMeta(cm).getAttributes().put(OperatorContext.PROCESSING_MODE, processingMode);
dag.addStream("connection", rip.output, cm.input);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class StatsTest method baseTestForQueueSize.
@SuppressWarnings("SleepWhileInLoop")
private void baseTestForQueueSize(int maxTuples, TestCollectorStatsListener statsListener, DAG.Locality locality) throws Exception {
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/baseTestForQueueSize").getAbsolutePath();
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 200);
TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
testOper.setMaxTuples(maxTuples);
TestCollector collector = dag.addOperator("Collector", new TestCollector());
if (statsListener != null) {
dag.setOperatorAttribute(collector, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { statsListener }));
}
dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(locality);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
StreamingContainerManager dnmgr = lc.getStreamingContainerManager();
Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
for (PTOperator p : operatorMap.values()) {
StramTestSupport.waitForActivation(lc, p);
}
long startTms = System.currentTimeMillis();
if (statsListener != null) {
while (statsListener.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
Thread.sleep(300);
LOG.debug("Waiting for stats");
}
} else {
while (collector.collectorOperatorStats.isEmpty() && (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms)) {
Thread.sleep(300);
LOG.debug("Waiting for stats");
}
}
if (statsListener != null) {
statsListener.validateStats();
} else {
collector.validateStats();
}
lc.shutdown();
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class TupleRecorderTest method testRecordingFlow.
@Test
public void testRecordingFlow() throws Exception {
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir.getAbsolutePath(), null));
dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, "file://" + testWorkDir.getAbsolutePath());
// 1KB per part
dag.getAttributes().put(LogicalPlan.TUPLE_RECORDING_PART_FILE_SIZE, 1024);
TestGeneratorInputOperator op1 = dag.addOperator("op1", TestGeneratorInputOperator.class);
GenericTestOperator op2 = dag.addOperator("op2", GenericTestOperator.class);
GenericTestOperator op3 = dag.addOperator("op3", GenericTestOperator.class);
// emit every 100 msec
op1.setEmitInterval(100);
//.setInline(true);
dag.addStream("stream1", op1.outport, op2.inport1);
//.setInline(true);
dag.addStream("stream2", op2.outport1, op3.inport1);
final StramLocalCluster localCluster = new StramLocalCluster(dag);
localCluster.runAsync();
final PTOperator ptOp2 = localCluster.findByLogicalNode(dag.getMeta(op2));
StramTestSupport.waitForActivation(localCluster, ptOp2);
testRecordingOnOperator(localCluster, ptOp2);
final PTOperator ptOp1 = localCluster.findByLogicalNode(dag.getMeta(op1));
StramTestSupport.waitForActivation(localCluster, ptOp1);
testRecordingOnOperator(localCluster, ptOp1);
localCluster.shutdown();
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class AtLeastOnceTest method testOperatorRecovery.
@Test
public void testOperatorRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
int maxTuples = 30;
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
asyncFSStorageAgent.setSyncCheckpoint(true);
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
cm.setSimulateFailure(true);
dag.addStream("connection", rip.output, cm.input);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}
Aggregations