Examples with GenericTestOperator com.datatorrent.stram.engine.GenericTestOperator used on opensource projects

Example 36 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LatencyTest method setup.

@Before
public void setup() {
    dag = StramTestSupport.createDAG(testMeta);
    dag.setAttribute(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS, windowWidthMillis);
    dag.setAttribute(Context.DAGContext.HEARTBEAT_TIMEOUT_MILLIS, heartbeatTimeoutMillis);
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1.output1", o1.outport1, o3.inport1);
    dag.addStream("o2.output1", o2.outport1, o3.inport2);
    scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
    o3p1 = plan.getOperators(dag.getMeta(o3)).get(0);
}

Example 37 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class PhysicalPlanTest method testContainerSizeWithPartitioning.

@Test
public void testContainerSizeWithPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 10);
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 5, plan.getContainers().size());
    PTContainer container;
    for (int i = 0; i < 5; i++) {
        container = plan.getContainers().get(i);
        if (container.getOperators().size() == 1) {
            Assert.assertEquals("container memory is 1536 for container :" + container, 1536, container.getRequiredMemoryMB());
        }
        if (container.getOperators().size() == 2) {
            Assert.assertEquals("container memory is 2048 for container :" + container, 2048, container.getRequiredMemoryMB());
        }
    }
}

Example 38 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenStreamIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenStreamIsRemoved() {
    // Remove Stream and check if persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    ((LogicalPlan.StreamMeta) stream).remove();
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator should be removed from dag after stream.remove", null, persistOperatorMeta);
}

Example 39 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsAdded.

@Test
public void testPersistStreamOperatorIsAdded() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    dag.validate();
}

Example 40 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class StreamPersistanceTests method testaddStreamThrowsExceptionOnInvalidLoggerType.

@Test
public void testaddStreamThrowsExceptionOnInvalidLoggerType() {
    // Test Logger with non-optional output ports
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    TestOperatorWithOutputPorts persister = new TestOperatorWithOutputPorts();
    try {
        stream.persistUsing("persister", persister, persister.inputPort);
        Assert.fail("should throw Illegal argument exception: Persist operator has non optional output ports");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test already added operator passed
    TestOperatorWithOutputPorts persister1 = new TestOperatorWithOutputPorts();
    try {
        stream.persistUsing("Stream1_persister", persister1, persister1.inputPort);
        Assert.fail("should throw exception that Stream1_persister object was already added");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test persist operator without any input ports
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
    TestOperatorWithoutInputPorts logger2 = new TestOperatorWithoutInputPorts();
    try {
        stream.persistUsing("Stream1_persister", logger2);
        Assert.fail("should throw Illegal argument exception: persist operator should have input ports");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test persist operator with more than one input port as non-optional
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
    TestOperatorWithMultipleNonOptionalInputPorts persister3 = new TestOperatorWithMultipleNonOptionalInputPorts();
    try {
        stream.persistUsing("Stream1_persister", persister3);
        Assert.fail("should throw Illegal argument exception: persist operator should have at most 1 non-optional input port");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
}