Examples with TestGeneratorInputOperator com.datatorrent.stram.engine.TestGeneratorInputOperator used on opensource projects

Example 31 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class TupleRecorderTest method testRecordingFlow.

@Test
public void testRecordingFlow() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir.getAbsolutePath(), null));
    dag.getAttributes().put(LogicalPlan.APPLICATION_PATH, "file://" + testWorkDir.getAbsolutePath());
    // 1KB per part
    dag.getAttributes().put(LogicalPlan.TUPLE_RECORDING_PART_FILE_SIZE, 1024);
    TestGeneratorInputOperator op1 = dag.addOperator("op1", TestGeneratorInputOperator.class);
    GenericTestOperator op2 = dag.addOperator("op2", GenericTestOperator.class);
    GenericTestOperator op3 = dag.addOperator("op3", GenericTestOperator.class);
    // emit every 100 msec
    op1.setEmitInterval(100);
    // .setInline(true);
    dag.addStream("stream1", op1.outport, op2.inport1);
    // .setInline(true);
    dag.addStream("stream2", op2.outport1, op3.inport1);
    final StramLocalCluster localCluster = new StramLocalCluster(dag);
    localCluster.runAsync();
    final PTOperator ptOp2 = localCluster.findByLogicalNode(dag.getMeta(op2));
    StramTestSupport.waitForActivation(localCluster, ptOp2);
    testRecordingOnOperator(localCluster, ptOp2);
    final PTOperator ptOp1 = localCluster.findByLogicalNode(dag.getMeta(op1));
    StramTestSupport.waitForActivation(localCluster, ptOp1);
    testRecordingOnOperator(localCluster, ptOp1);
    localCluster.shutdown();
}

Example 32 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class PhysicalPlanTest method testStaticPartitioning.

@Test
public void testStaticPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    TestGeneratorInputOperator node0 = dag.addOperator("node0", TestGeneratorInputOperator.class);
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    PartitioningTestOperator partitioned = dag.addOperator("partitioned", PartitioningTestOperator.class);
    partitioned.setPartitionCount(partitioned.partitionKeys.length);
    GenericTestOperator singleton1 = dag.addOperator("singleton1", GenericTestOperator.class);
    GenericTestOperator singleton2 = dag.addOperator("singleton2", GenericTestOperator.class);
    dag.addStream("n0.inport1", node0.outport, node1.inport1);
    dag.addStream("n1.outport1", node1.outport1, partitioned.inport1, partitioned.inportWithCodec);
    dag.addStream("mergeStream", partitioned.outport1, singleton1.inport1, singleton2.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    OperatorMeta partitionedMeta = dag.getMeta(partitioned);
    dag.validate();
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    Assert.assertEquals("number of containers", 2, plan.getContainers().size());
    Assert.assertNotNull("partition map", partitioned.partitions);
    Assert.assertEquals("partition map " + partitioned.partitions, 3, partitioned.partitions.size());
    List<PTOperator> n2Instances = plan.getOperators(partitionedMeta);
    Assert.assertEquals("partition instances " + n2Instances, partitioned.partitionKeys.length, n2Instances.size());
    for (int i = 0; i < n2Instances.size(); i++) {
        PTOperator po = n2Instances.get(i);
        Map<String, PTInput> inputsMap = new HashMap<>();
        for (PTInput input : po.getInputs()) {
            inputsMap.put(input.portName, input);
            Assert.assertEquals("partitions " + input, Sets.newHashSet(partitioned.partitionKeys[i]), input.partitions.partitions);
        // Assert.assertEquals("codec " + input.logicalStream, PartitioningTestStreamCodec.class, input.logicalStream.getCodecClass());
        }
        Assert.assertEquals("number inputs " + inputsMap, Sets.newHashSet(PartitioningTestOperator.IPORT1, PartitioningTestOperator.INPORT_WITH_CODEC), inputsMap.keySet());
    }
    Collection<PTOperator> unifiers = plan.getMergeOperators(partitionedMeta);
    Assert.assertEquals("number unifiers " + partitionedMeta, 0, unifiers.size());
}

Example 33 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamPersistanceTests method testaddStreamThrowsExceptionOnInvalidInputPortForLoggerType.

@Test
public void testaddStreamThrowsExceptionOnInvalidInputPortForLoggerType() {
    // Test for input port belonging to different object
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    try {
        stream.persistUsing("Stream1_persister", persister, persister1.inport);
        Assert.fail("should throw Illegal argument exception: Port passed does not belong to operator class");
    } catch (IllegalArgumentException e) {
    // all good
    }
    // Remove persist operator from dag
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
}

Example 34 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenAllSinksAreRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenAllSinksAreRemoved() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    // Check stream persister is added to the dag
    Assert.assertNotNull("Stream persister operator should be present", dag.getOperatorMeta("Stream1_persister"));
    // remove sink operators x1, x2, x3 from dag and check that persister
    // operator is removed
    dag.removeOperator(x1);
    dag.removeOperator(x2);
    dag.removeOperator(x3);
    Assert.assertNull("Persister operator should have been removed after all sinks are removed", dag.getOperatorMeta("Stream1_persister"));
}

Example 35 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanConfigurationTest method testTestTupleClassAttrSetFromConfig.

@Test
public void testTestTupleClassAttrSetFromConfig() {
    Configuration conf = new Configuration(false);
    conf.set(StreamingApplication.APEX_PREFIX + "operator.o2.port.schemaRequiredPort.attr.TUPLE_CLASS", "com.datatorrent.stram.plan.logical.LogicalPlanConfigurationTest$TestSchema");
    StreamingApplication streamingApplication = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            TestGeneratorInputOperator o1 = dag.addOperator("o1", new TestGeneratorInputOperator());
            SchemaTestOperator o2 = dag.addOperator("o2", new SchemaTestOperator());
            dag.addStream("stream", o1.outport, o2.schemaRequiredPort);
        }
    };
    LogicalPlan dag = new LogicalPlan();
    LogicalPlanConfiguration lpc = new LogicalPlanConfiguration(conf);
    lpc.prepareDAG(dag, streamingApplication, "app");
    dag.validate();
}