Examples with StreamMeta com.datatorrent.api.DAG.StreamMeta used on opensource projects

Example 6 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenStreamIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenStreamIsRemoved() {
    // Remove Stream and check if persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    ((LogicalPlan.StreamMeta) stream).remove();
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator should be removed from dag after stream.remove", null, persistOperatorMeta);
}

Example 7 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenSinkIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenSinkIsRemoved() {
    // Remove sink and check if corresponding persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    TestReceiverOperator persister2 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    stream.persistUsing("Stream1_x1_persister", persister1, persister1.inport, x1.inport1);
    stream.persistUsing("Stream1_x2_persister", persister2, persister2.inport, x2.inport1);
    // Check 3 persist operators are added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator not added to dag ", persister1, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
    dag.removeOperator(x1);
    // Check persister for x1 is removed
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator should be removed from dag after sink is removed", null, persistOperatorMeta);
    // Check other persisters are unchanged
    persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
}

Example 8 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsAddedPerSink.

@Test
public void testPersistStreamOperatorIsAddedPerSink() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    TestReceiverOperator persister2 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    stream.persistUsing("Stream1_x1_persister", persister1, persister1.inport, x1.inport1);
    stream.persistUsing("Stream1_x2_persister", persister2, persister2.inport, x2.inport1);
    // Check 3 persist operators are added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator not added to dag ", persister1, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
    dag.validate();
}

Example 9 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamWithFiltering.

@Test
public void testPersistStreamWithFiltering() throws ClassNotFoundException, IOException, InterruptedException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    PassThruOperatorWithCodec passThru = dag.addOperator("PassThrough", new PassThruOperatorWithCodec(2));
    TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    TestPersistanceOperator console1 = new TestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, passThru.input);
    s.persistUsing("Stream1_persister", console1, console1.inport);
    dag.addStream("Stream2", passThru.output, console.inport);
    runLocalClusterAndValidate(dag, console, console1);
}

Example 10 with StreamMeta

use of com.datatorrent.api.DAG.StreamMeta in project apex-core by apache.

the class StreamPersistanceTests method testaddStreamThrowsExceptionOnInvalidInputPortForLoggerType.

@Test
public void testaddStreamThrowsExceptionOnInvalidInputPortForLoggerType() {
    // Test for input port belonging to different object
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    try {
        stream.persistUsing("Stream1_persister", persister, persister1.inport);
        Assert.fail("should throw Illegal argument exception: Port passed does not belong to operator class");
    } catch (IllegalArgumentException e) {
    // all good
    }
    // Remove persist operator from dag
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
}