Examples with TestGeneratorInputOperator com.datatorrent.stram.engine.TestGeneratorInputOperator used on opensource projects

Example 16 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanTest method testDeleteOperator.

@Test
public void testDeleteOperator() {
    TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("s0", input.outport, o1.inport1);
    StreamMeta s1 = dag.addStream("s1", o1.outport1, o2.inport1);
    dag.validate();
    Assert.assertEquals("", 3, dag.getAllOperators().size());
    dag.removeOperator(o2);
    s1.remove();
    dag.validate();
    Assert.assertEquals("", 2, dag.getAllOperators().size());
}

Example 17 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanTest method testOperatorAnnotation.

@Test
public void testOperatorAnnotation() {
    TestGeneratorInputOperator input = dag.addOperator("input1", TestGeneratorInputOperator.class);
    TestOperatorAnnotationOperator operator = dag.addOperator("operator1", TestOperatorAnnotationOperator.class);
    dag.addStream("Connection", input.outport, operator.input1);
    dag.setOperatorAttribute(operator, OperatorContext.PARTITIONER, new StatelessPartitioner<TestOperatorAnnotationOperator>(2));
    try {
        dag.validate();
        Assert.fail("should raise operator is not partitionable for operator1");
    } catch (ValidationException e) {
        Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
    }
    dag.setOperatorAttribute(operator, OperatorContext.PARTITIONER, null);
    dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, true);
    try {
        dag.validate();
        Assert.fail("should raise operator is not partitionable for operator1");
    } catch (ValidationException e) {
        Assert.assertEquals("", "Operator " + dag.getMeta(operator).getName() + " is not partitionable but PARTITION_PARALLEL attribute is set", e.getMessage());
    }
    dag.setInputPortAttribute(operator.input1, PortContext.PARTITION_PARALLEL, false);
    dag.validate();
    dag.removeOperator(operator);
    TestOperatorAnnotationOperator2 operator2 = dag.addOperator("operator2", TestOperatorAnnotationOperator2.class);
    try {
        dag.validate();
        Assert.fail("should raise operator is not partitionable for operator2");
    } catch (ValidationException e) {
        Assert.assertEquals("Operator " + dag.getMeta(operator2).getName() + " provides partitioning capabilities but the annotation on the operator class declares it non partitionable!", e.getMessage());
    }
}

Example 18 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanTest method testAffinityRulesDagValidation.

@Test
public void testAffinityRulesDagValidation() {
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1).setLocality(Locality.THREAD_LOCAL);
    StreamMeta stream2 = dag.addStream("stream2", o2.outport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O3");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    // Locality conflicts with affinity rules case:
    AffinityRule rule2 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule2);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O2 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change Stream2 locality to Node to check if validation passes
    stream2.setLocality(Locality.RACK_LOCAL);
    dag.validate();
    // Add anti-affinity rule conflicting with rule1
    AffinityRule rule3 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O3");
    rules.add(rule3);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O1 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change rule1 to Rack local to see if dag validation passes
    rules.clear();
    rule1.setLocality(Locality.RACK_LOCAL);
    rules.add(rule1);
    rules.add(rule2);
    rules.add(rule3);
    dag.validate();
    // Add conflicting rules and set relaxLocality for one rule
    AffinityRule rule4 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, true, "O1", "O2");
    rules.add(rule4);
    dag.validate();
    // Set conflicting host locality and check if it fails validation
    rules.clear();
    AffinityRule rule = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host1) conflict with anti-affinity rules", e.getMessage());
    }
    // Set conflicting affinity and different host locality for node-local
    // operators
    rules.clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host2) conflicts with affinity rules", e.getMessage());
    }
    // Check affinity Thread local validation for non-connected operators
    dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET).getAffinityRules().clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.THREAD_LOCAL, false, "O1", "O3");
    rules.add(rule);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Affinity rule specified THREAD_LOCAL affinity for operators O1 & O3 which are not connected by stream", e.getMessage());
    }
    // Check indirect conflict
    dag = new LogicalPlan();
    o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    o2 = dag.addOperator("O2", new GenericTestOperator());
    o3 = dag.addOperator("O3", new GenericTestOperator());
    GenericTestOperator o4 = dag.addOperator("O4", new GenericTestOperator());
    GenericTestOperator o5 = dag.addOperator("O5", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1, o3.inport1).setLocality(Locality.NODE_LOCAL);
    dag.addStream("stream2", o3.outport1, o4.inport1);
    dag.addStream("stream3", o2.outport1, o5.inport1);
    rules.clear();
    // O3 and O5 cannot have NODE_LOCAL anti-affinity now, since they already have NODE_LOCAL affinity
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O5"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O3", "O5"));
    ruleSet = new AffinityRulesSet();
    ruleSet.setAffinityRules(rules);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    try {
        dag.validate();
        Assert.fail("dag validation should fail due to conflicting affinity rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O3 & O5 conflicts with affinity rules or Stream locality", e.getMessage());
    }
}

Example 19 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanTest method testInputPortHiding.

@Test
public void testInputPortHiding() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    Operator2 operator2 = dag.addOperator("operator2", new Operator2());
    dag.addStream("Stream1", input1.outport, operator2.input);
    dag.validate();
}

Example 20 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class LogicalPlanTest method testCycleDetectionWithDelay.

@Test
public void testCycleDetectionWithDelay() {
    TestGeneratorInputOperator opA = dag.addOperator("A", TestGeneratorInputOperator.class);
    GenericTestOperator opB = dag.addOperator("B", GenericTestOperator.class);
    GenericTestOperator opC = dag.addOperator("C", GenericTestOperator.class);
    GenericTestOperator opD = dag.addOperator("D", GenericTestOperator.class);
    DefaultDelayOperator<Object> opDelay = dag.addOperator("opDelay", new DefaultDelayOperator<>());
    DefaultDelayOperator<Object> opDelay2 = dag.addOperator("opDelay2", new DefaultDelayOperator<>());
    dag.addStream("AtoB", opA.outport, opB.inport1);
    dag.addStream("BtoC", opB.outport1, opC.inport1);
    dag.addStream("CtoD", opC.outport1, opD.inport1);
    dag.addStream("CtoDelay", opC.outport2, opDelay.input);
    dag.addStream("DtoDelay", opD.outport1, opDelay2.input);
    dag.addStream("DelayToB", opDelay.output, opB.inport2);
    dag.addStream("Delay2ToC", opDelay2.output, opC.inport2);
    LogicalPlan.ValidationContext vc = new LogicalPlan.ValidationContext();
    dag.findStronglyConnected(dag.getMeta(opA), vc);
    Assert.assertEquals("No invalid cycle", Collections.emptyList(), vc.invalidCycles);
    Set<OperatorMeta> exp = Sets.newHashSet(dag.getMeta(opDelay2), dag.getMeta(opDelay), dag.getMeta(opC), dag.getMeta(opB), dag.getMeta(opD));
    Assert.assertEquals("cycle", exp, vc.stronglyConnected.get(0));
}