Examples with GenericTestOperator com.datatorrent.stram.engine.GenericTestOperator used on opensource projects

Example 81 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class DelayOperatorTest method testCheckpointUpdate.

@Test
public void testCheckpointUpdate() {
    LogicalPlan dag = StramTestSupport.createDAG(testMeta);
    TestGeneratorInputOperator opA = dag.addOperator("A", TestGeneratorInputOperator.class);
    GenericTestOperator opB = dag.addOperator("B", GenericTestOperator.class);
    GenericTestOperator opC = dag.addOperator("C", GenericTestOperator.class);
    GenericTestOperator opD = dag.addOperator("D", GenericTestOperator.class);
    DefaultDelayOperator<Object> opDelay = dag.addOperator("opDelay", new DefaultDelayOperator<>());
    dag.addStream("AtoB", opA.outport, opB.inport1);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input);
    dag.addStream("DelayToB", opDelay.output, opB.inport2);
    dag.validate();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    // set all operators as active to enable recovery window id update
    for (PTOperator oper : plan.getAllOperators().values()) {
        oper.setState(PTOperator.State.ACTIVE);
    }
    Clock clock = new SystemClock();
    PTOperator opA1 = plan.getOperators(dag.getMeta(opA)).get(0);
    PTOperator opB1 = plan.getOperators(dag.getMeta(opB)).get(0);
    PTOperator opC1 = plan.getOperators(dag.getMeta(opC)).get(0);
    PTOperator opDelay1 = plan.getOperators(dag.getMeta(opDelay)).get(0);
    PTOperator opD1 = plan.getOperators(dag.getMeta(opD)).get(0);
    Checkpoint cp3 = new Checkpoint(3L, 0, 0);
    Checkpoint cp5 = new Checkpoint(5L, 0, 0);
    Checkpoint cp4 = new Checkpoint(4L, 0, 0);
    opB1.checkpoints.add(cp3);
    opC1.checkpoints.add(cp3);
    opC1.checkpoints.add(cp4);
    opDelay1.checkpoints.add(cp3);
    opDelay1.checkpoints.add(cp5);
    opD1.checkpoints.add(cp5);
    // construct grouping that would be supplied through LogicalPlan
    Set<OperatorMeta> stronglyConnected = Sets.newHashSet(dag.getMeta(opB), dag.getMeta(opC), dag.getMeta(opDelay));
    Map<OperatorMeta, Set<OperatorMeta>> groups = new HashMap<>();
    for (OperatorMeta om : stronglyConnected) {
        groups.put(om, stronglyConnected);
    }
    UpdateCheckpointsContext ctx = new UpdateCheckpointsContext(clock, false, groups);
    scm.updateRecoveryCheckpoints(opB1, ctx, false);
    Assert.assertEquals("checkpoint " + opA1, Checkpoint.INITIAL_CHECKPOINT, opA1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opB1, cp3, opC1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opC1, cp3, opC1.getRecoveryCheckpoint());
    Assert.assertEquals("checkpoint " + opD1, cp5, opD1.getRecoveryCheckpoint());
}

Example 82 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanTest method testAtMostOnceProcessingModeValidation.

@Test
public void testAtMostOnceProcessingModeValidation() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
    GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
    dag.setOperatorAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
    dag.addStream("input1.outport", input1.outport, amoOper.inport1);
    dag.addStream("input2.outport", input2.outport, amoOper.inport2);
    GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
    dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
    dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
    try {
        dag.validate();
        Assert.fail("Exception expected for " + outputOper);
    } catch (ValidationException ve) {
        Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/AT_MOST_ONCE");
    }
    dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, null);
    dag.validate();
    OperatorMeta outputOperOm = dag.getMeta(outputOper);
    Assert.assertEquals("" + outputOperOm.getAttributes(), Operator.ProcessingMode.AT_MOST_ONCE, outputOperOm.getValue(OperatorContext.PROCESSING_MODE));
}

Example 83 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanTest method testCheckpointableWithinAppWindowAnnotation.

/*
  @Test
  public void testStreamCodec() throws Exception {
    TestGeneratorInputOperator input = dag.addOperator("input", TestGeneratorInputOperator.class);
    GenericTestOperator gto1 = dag.addOperator("gto1", GenericTestOperator.class);
    StreamMeta stream1 = dag.addStream("s1", input.outport, gto1.inport1);
    StreamCodec<?> codec1 = new TestStreamCodec();
    dag.setInputPortAttribute(gto1.inport1, PortContext.STREAM_CODEC, codec1);
    dag.validate();
    //Assert.assertEquals("Stream codec not set", stream1.getStreamCodec(), codec1);

    GenericTestOperator gto2 = dag.addOperator("gto2", GenericTestOperator.class);
    GenericTestOperator gto3 = dag.addOperator("gto3", GenericTestOperator.class);
    StreamMeta stream2 = dag.addStream("s2", gto1.outport1, gto2.inport1, gto3.inport1);
    dag.setInputPortAttribute(gto2.inport1, PortContext.STREAM_CODEC, codec1);
    try {
      dag.validate();
    } catch (ValidationException e) {
      String msg = e.getMessage();
      if (!msg.startsWith("Stream codec not set on input port") || !msg.contains("gto3")
              || !msg.contains(codec1.toString()) || !msg.endsWith("was specified on another port")) {
        Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
      }
    }

    dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
    dag.validate();
    //Assert.assertEquals("Stream codec not set", stream2.getStreamCodec(), codec1);

    StreamCodec<?> codec2 = new TestStreamCodec();
    dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec2);
    try {
      dag.validate();
    } catch (ValidationException e) {
      String msg = e.getMessage();
      if (!msg.startsWith("Conflicting stream codec set on input port") || !msg.contains("gto3")
              || !msg.contains(codec2.toString()) || !msg.endsWith("was specified on another port")) {
        Assert.fail(String.format("LogicalPlan validation error msg: %s", msg));
      }
    }

    dag.setInputPortAttribute(gto3.inport1, PortContext.STREAM_CODEC, codec1);
    TestPortCodecOperator pco = dag.addOperator("pco", TestPortCodecOperator.class);
    StreamMeta stream3 = dag.addStream("s3", gto2.outport1, pco.inport1);
    dag.validate();
    //Assert.assertEquals("Stream codec class not set", stream3.getCodecClass(), TestStreamCodec.class);

    dag.setInputPortAttribute(pco.inport1, PortContext.STREAM_CODEC, codec2);
    dag.validate();
    //Assert.assertEquals("Stream codec not set", stream3.getStreamCodec(), codec2);
  }
  */
@Test
public void testCheckpointableWithinAppWindowAnnotation() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    dag.addStream("Stream1", input1.outport, x.inport1);
    dag.setOperatorAttribute(x, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
    dag.setOperatorAttribute(x, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
    dag.validate();
    TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
    CheckpointableWithinAppWindowOperator y = dag.addOperator("y", new CheckpointableWithinAppWindowOperator());
    dag.addStream("Stream2", input2.outport, y.inport1);
    dag.setOperatorAttribute(y, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
    dag.setOperatorAttribute(y, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
    dag.validate();
    TestGeneratorInputOperator input3 = dag.addOperator("input3", TestGeneratorInputOperator.class);
    NotCheckpointableWithinAppWindowOperator z = dag.addOperator("z", new NotCheckpointableWithinAppWindowOperator());
    dag.addStream("Stream3", input3.outport, z.inport1);
    dag.setOperatorAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 15);
    dag.setOperatorAttribute(z, OperatorContext.APPLICATION_WINDOW_COUNT, 30);
    try {
        dag.validate();
        Assert.fail("should fail because chekpoint window count is not a factor of application window count");
    } catch (ValidationException e) {
    // expected
    }
    dag.setOperatorAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 30);
    dag.validate();
    dag.setOperatorAttribute(z, OperatorContext.CHECKPOINT_WINDOW_COUNT, 45);
    try {
        dag.validate();
        Assert.fail("should fail because chekpoint window count is not a factor of application window count");
    } catch (ValidationException e) {
    // expected
    }
}

Example 84 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanTest method testExactlyOnceProcessingModeValidation.

@Test
public void testExactlyOnceProcessingModeValidation() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    TestGeneratorInputOperator input2 = dag.addOperator("input2", TestGeneratorInputOperator.class);
    GenericTestOperator amoOper = dag.addOperator("amoOper", GenericTestOperator.class);
    dag.setOperatorAttribute(amoOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.EXACTLY_ONCE);
    dag.addStream("input1.outport", input1.outport, amoOper.inport1);
    dag.addStream("input2.outport", input2.outport, amoOper.inport2);
    GenericTestOperator outputOper = dag.addOperator("outputOper", GenericTestOperator.class);
    dag.addStream("aloOper.outport1", amoOper.outport1, outputOper.inport1);
    try {
        dag.validate();
        Assert.fail("Exception expected for " + outputOper);
    } catch (ValidationException ve) {
        Assert.assertEquals("", ve.getMessage(), "Processing mode for outputOper should be AT_MOST_ONCE for source amoOper/EXACTLY_ONCE");
    }
    dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_LEAST_ONCE);
    try {
        dag.validate();
        Assert.fail("Exception expected for " + outputOper);
    } catch (ValidationException ve) {
        Assert.assertEquals("", ve.getMessage(), "Processing mode outputOper/AT_LEAST_ONCE not valid for source amoOper/EXACTLY_ONCE");
    }
    // AT_MOST_ONCE is valid
    dag.setOperatorAttribute(outputOper, OperatorContext.PROCESSING_MODE, Operator.ProcessingMode.AT_MOST_ONCE);
    dag.validate();
}

Example 85 with GenericTestOperator

use of com.datatorrent.stram.engine.GenericTestOperator in project apex-core by apache.

the class LogicalPlanConfigurationTest method testLoadFromPropertiesFile.

@Test
public void testLoadFromPropertiesFile() throws IOException {
    Properties props = new Properties();
    String resourcePath = "/testTopology.properties";
    InputStream is = this.getClass().getResourceAsStream(resourcePath);
    if (is == null) {
        fail("Could not load " + resourcePath);
    }
    props.load(is);
    LogicalPlanConfiguration pb = new LogicalPlanConfiguration(new Configuration(false)).addFromProperties(props, null);
    LogicalPlan dag = new LogicalPlan();
    pb.populateDAG(dag);
    dag.validate();
    assertEquals("number of operator confs", 5, dag.getAllOperators().size());
    assertEquals("number of root operators", 1, dag.getRootOperators().size());
    StreamMeta s1 = dag.getStream("n1n2");
    assertNotNull(s1);
    assertTrue("n1n2 locality", DAG.Locality.CONTAINER_LOCAL == s1.getLocality());
    OperatorMeta operator3 = dag.getOperatorMeta("operator3");
    assertEquals("operator3.classname", GenericTestOperator.class, operator3.getOperator().getClass());
    GenericTestOperator doperator3 = (GenericTestOperator) operator3.getOperator();
    assertEquals("myStringProperty " + doperator3, "myStringPropertyValueFromTemplate", doperator3.getMyStringProperty());
    assertFalse("booleanProperty " + doperator3, doperator3.booleanProperty);
    OperatorMeta operator4 = dag.getOperatorMeta("operator4");
    GenericTestOperator doperator4 = (GenericTestOperator) operator4.getOperator();
    assertEquals("myStringProperty " + doperator4, "overrideOperator4", doperator4.getMyStringProperty());
    assertEquals("setterOnlyOperator4 " + doperator4, "setterOnlyOperator4", doperator4.propertySetterOnly);
    assertTrue("booleanProperty " + doperator4, doperator4.booleanProperty);
    StreamMeta input1 = dag.getStream("inputStream");
    assertNotNull(input1);
    Assert.assertEquals("input1 source", dag.getOperatorMeta("inputOperator"), input1.getSource().getOperatorMeta());
    Set<OperatorMeta> targetNodes = Sets.newHashSet();
    for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
        targetNodes.add(targetPort.getOperatorMeta());
    }
    Assert.assertEquals("input1 target ", Sets.newHashSet(dag.getOperatorMeta("operator1"), operator3, operator4), targetNodes);
}