Examples with OperatorMeta com.datatorrent.api.DAG.OperatorMeta used on opensource projects

Example 1 with OperatorMeta

use of com.datatorrent.api.DAG.OperatorMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsAdded.

@Test
public void testPersistStreamOperatorIsAdded() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    dag.validate();
}

Example 2 with OperatorMeta

use of com.datatorrent.api.DAG.OperatorMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenStreamIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenStreamIsRemoved() {
    // Remove Stream and check if persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    ((LogicalPlan.StreamMeta) stream).remove();
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator should be removed from dag after stream.remove", null, persistOperatorMeta);
}

Example 3 with OperatorMeta

use of com.datatorrent.api.DAG.OperatorMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenSinkIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenSinkIsRemoved() {
    // Remove sink and check if corresponding persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    TestReceiverOperator persister2 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    stream.persistUsing("Stream1_x1_persister", persister1, persister1.inport, x1.inport1);
    stream.persistUsing("Stream1_x2_persister", persister2, persister2.inport, x2.inport1);
    // Check 3 persist operators are added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator not added to dag ", persister1, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
    dag.removeOperator(x1);
    // Check persister for x1 is removed
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator should be removed from dag after sink is removed", null, persistOperatorMeta);
    // Check other persisters are unchanged
    persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
}

Example 4 with OperatorMeta

use of com.datatorrent.api.DAG.OperatorMeta in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsAddedPerSink.

@Test
public void testPersistStreamOperatorIsAddedPerSink() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x1 = dag.addOperator("x1", new GenericTestOperator());
    GenericTestOperator x2 = dag.addOperator("x2", new GenericTestOperator());
    GenericTestOperator x3 = dag.addOperator("x3", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    TestReceiverOperator persister1 = new TestReceiverOperator();
    TestReceiverOperator persister2 = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x1.inport1, x2.inport1, x3.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    stream.persistUsing("Stream1_x1_persister", persister1, persister1.inport, x1.inport1);
    stream.persistUsing("Stream1_x2_persister", persister2, persister2.inport, x2.inport1);
    // Check 3 persist operators are added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x1_persister");
    assertEquals("Persist operator not added to dag ", persister1, persistOperatorMeta.getOperator());
    persistOperatorMeta = dag.getOperatorMeta("Stream1_x2_persister");
    assertEquals("Persist operator not added to dag ", persister2, persistOperatorMeta.getOperator());
    dag.validate();
}

Example 5 with OperatorMeta

use of com.datatorrent.api.DAG.OperatorMeta in project beam by apache.

the class ApexRunnerTest method testConfigProperties.

@Test
public void testConfigProperties() throws Exception {
    String operName = "testProperties";
    ApexPipelineOptions options = PipelineOptionsFactory.create().as(ApexPipelineOptions.class);
    // default configuration from class path
    Pipeline p = Pipeline.create();
    Create.Values<Void> empty = Create.empty(VoidCoder.of());
    p.apply(operName, empty);
    DAG dag = TestApexRunner.translate(p, options);
    OperatorMeta t1Meta = dag.getOperatorMeta(operName);
    Assert.assertNotNull(t1Meta);
    Assert.assertEquals(new Integer(32), t1Meta.getValue(OperatorContext.MEMORY_MB));
    File tmp = File.createTempFile("beam-runners-apex-", ".properties");
    tmp.deleteOnExit();
    Properties props = new Properties();
    props.setProperty("apex.operator." + operName + ".attr.MEMORY_MB", "64");
    try (FileOutputStream fos = new FileOutputStream(tmp)) {
        props.store(fos, "");
    }
    options.setConfigFile(tmp.getAbsolutePath());
    dag = TestApexRunner.translate(p, options);
    tmp.delete();
    t1Meta = dag.getOperatorMeta(operName);
    Assert.assertNotNull(t1Meta);
    Assert.assertEquals(new Integer(64), t1Meta.getValue(OperatorContext.MEMORY_MB));
}