Examples with CollectorOperator com.datatorrent.stram.engine.ProcessingModeTests.CollectorOperator used on opensource projects

Example 1 with CollectorOperator

use of com.datatorrent.stram.engine.ProcessingModeTests.CollectorOperator in project apex-core by apache.

the class AtLeastOnceTest method testInlineOperatorsRecovery.

@Test
public void testInlineOperatorsRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    //dag.getAttributes().get(DAG.HEARTBEAT_INTERVAL_MILLIS, 400);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    //    for (Long l: collection) {
    //      logger.debug(Codec.getStringWindowId(l));
    //    }
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 2 with CollectorOperator

use of com.datatorrent.stram.engine.ProcessingModeTests.CollectorOperator in project apex-core by apache.

the class AtLeastOnceTest method testOperatorRecovery.

@Test
public void testOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 3 with CollectorOperator

use of com.datatorrent.stram.engine.ProcessingModeTests.CollectorOperator in project apex-core by apache.

the class AtLeastOnceTest method testInputOperatorRecovery.

@Test
public void testInputOperatorRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testInputOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    dag.addStream("connection", rip.output, cm.input);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 4 with CollectorOperator

use of com.datatorrent.stram.engine.ProcessingModeTests.CollectorOperator in project apex-core by apache.

the class OiOStreamTest method validatePositiveOiO.

@Test
public void validatePositiveOiO() {
    logger.info("Checking the logic for sanity checking of OiO");
    LogicalPlan plan = new LogicalPlan();
    RecoverableInputOperator inputOperator = plan.addOperator("IntegerGenerator", new RecoverableInputOperator());
    CollectorOperator outputOperator = plan.addOperator("IntegerCollector", new CollectorOperator());
    plan.addStream("PossibleOiO", inputOperator.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
    try {
        plan.validate();
        Assert.assertTrue("OiO validation", true);
    } catch (ConstraintViolationException ex) {
        Assert.fail("OIO Single InputPort");
    }
}