use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class StreamPersistanceTests method testPartitionedPersistOperator.
@Test
public void testPartitionedPersistOperator() throws ClassNotFoundException, IOException {
AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
PartitionedTestOperatorWithFiltering passThru = dag.addOperator("partition", new PartitionedTestOperatorWithFiltering());
final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input);
dag.setInputPortAttribute(passThru.input, PortContext.STREAM_CODEC, new TestPartitionCodec());
s.persistUsing("persister", console1, console1.inport);
dag.setInputPortAttribute(console1.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
dag.addStream("Stream2", passThru.output, console.inport);
final StramLocalCluster lc = new StramLocalCluster(dag);
new Thread("LocalClusterController") {
@Override
public void run() {
long startTms = System.currentTimeMillis();
long timeout = 100000L;
try {
while (System.currentTimeMillis() - startTms < timeout) {
if (console1.results.size() < 6) {
Thread.sleep(10);
} else {
break;
}
}
} catch (Exception ex) {
throw Throwables.propagate(ex);
} finally {
lc.shutdown();
}
}
}.start();
lc.run();
try {
// Values as per persist operator's partition keys should be picked up
Integer[] expectedResult = { 0, 4, 8, 12, 16, 20 };
for (int i = 0; i < expectedResult.length; i++) {
logger.debug(console1.results.get(i) + " " + expectedResult[i]);
assertEquals("Mismatch observed for tuple ", expectedResult[i], console1.results.get(i));
}
} finally {
console1.results.clear();
console.results.clear();
}
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class NodeTest method testStreamingWindowGenericNode.
@Ignore
@Test
public void testStreamingWindowGenericNode() throws Exception {
LogicalPlan dag = new LogicalPlan();
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 10);
dag.addOperator("GenericOperator", new TestGenericOperator());
final StramLocalCluster lc = new StramLocalCluster(dag);
lc.run(2000);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class AtLeastOnceTest method testInputOperatorRecovery.
@Test
public void testInputOperatorRecovery() throws Exception {
RecoverableInputOperator.initGenTuples();
CollectorOperator.collection.clear();
int maxTuples = 30;
LogicalPlan dag = new LogicalPlan();
String workingDir = new File("target/testInputOperatorRecovery").getAbsolutePath();
AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
asyncFSStorageAgent.setSyncCheckpoint(true);
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
rip.setMaximumTuples(maxTuples);
rip.setSimulateFailure(true);
CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
dag.addStream("connection", rip.output, cm.input);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.run();
Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class AutoMetricTest method testMetricPropagation.
/**
* Verify custom stats generated by operator are propagated and trigger repartition.
*
* @throws Exception
*/
@Test
@SuppressWarnings("SleepWhileInLoop")
public void testMetricPropagation() throws Exception {
dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
TestOperator testOper = dag.addOperator("TestOperator", TestOperator.class);
TestStatsListener sl = new TestStatsListener();
dag.setOperatorAttribute(testOper, OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) sl));
GenericTestOperator collector = dag.addOperator("Collector", new GenericTestOperator());
dag.addStream("TestTuples", testOper.outport, collector.inport1).setLocality(Locality.CONTAINER_LOCAL);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
long startTms = System.currentTimeMillis();
while (TestOperator.lastMetric == null && StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
Thread.sleep(300);
LOG.debug("Waiting for stats");
}
while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
if (sl.lastPropVal) {
break;
}
Thread.sleep(100);
LOG.debug("Waiting for property set");
}
lc.shutdown();
Assert.assertNotNull("metric received", TestOperator.lastMetric);
Assert.assertEquals("metric message", "interesting", TestOperator.lastMetric.message);
Assert.assertTrue("attribute defined stats listener called", TestOperator.lastMetric.attributeListenerCalled);
Assert.assertSame("single thread", TestOperator.definePartitionsThread, TestOperator.processStatsThread);
Assert.assertTrue("property set", sl.lastPropVal);
}
use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.
the class AutoMetricTest method testDefaultMetricsAggregator.
@Test
public void testDefaultMetricsAggregator() throws Exception {
LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(new Configuration());
TestGeneratorInputOperator inputOperator = dag.addOperator("input", TestGeneratorInputOperator.class);
CountDownLatch latch = new CountDownLatch(1);
OperatorAndAggregator o1 = dag.addOperator("o1", new OperatorAndAggregator(latch));
dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
dag.addStream("TestTuples", inputOperator.outport, o1.inport1);
lpc.prepareDAG(dag, null, "AutoMetricTest");
LogicalPlan.OperatorMeta o1meta = dag.getOperatorMeta("o1");
Assert.assertNotNull("default aggregator injected", o1meta.getMetricAggregatorMeta().getAggregator());
lpc.prepareDAG(dag, null, "AutoMetricTest");
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
latch.await();
Assert.assertEquals("progress", 1, o1.result.get("progress"));
lc.shutdown();
}
Aggregations