Examples with StreamMeta com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta used on opensource projects

Example 16 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanTest method testCycleDetection.

@Test
public void testCycleDetection() {
    //NodeConf operator1 = b.getOrAddNode("operator1");
    GenericTestOperator operator2 = dag.addOperator("operator2", GenericTestOperator.class);
    GenericTestOperator operator3 = dag.addOperator("operator3", GenericTestOperator.class);
    GenericTestOperator operator4 = dag.addOperator("operator4", GenericTestOperator.class);
    //NodeConf operator5 = b.getOrAddNode("operator5");
    //NodeConf operator6 = b.getOrAddNode("operator6");
    GenericTestOperator operator7 = dag.addOperator("operator7", GenericTestOperator.class);
    // strongly connect n2-n3-n4-n2
    dag.addStream("n2n3", operator2.outport1, operator3.inport1);
    dag.addStream("n3n4", operator3.outport1, operator4.inport1);
    dag.addStream("n4n2", operator4.outport1, operator2.inport1);
    // self referencing operator cycle
    StreamMeta n7n7 = dag.addStream("n7n7", operator7.outport1, operator7.inport1);
    try {
        n7n7.addSink(operator7.inport1);
        fail("cannot add to stream again");
    } catch (Exception e) {
    // expected, stream can have single input/output only
    }
    LogicalPlan.ValidationContext vc = new LogicalPlan.ValidationContext();
    dag.findStronglyConnected(dag.getMeta(operator7), vc);
    assertEquals("operator self reference", 1, vc.invalidCycles.size());
    assertEquals("operator self reference", 1, vc.invalidCycles.get(0).size());
    assertEquals("operator self reference", dag.getMeta(operator7), vc.invalidCycles.get(0).iterator().next());
    // 3 operator cycle
    vc = new LogicalPlan.ValidationContext();
    dag.findStronglyConnected(dag.getMeta(operator4), vc);
    assertEquals("3 operator cycle", 1, vc.invalidCycles.size());
    assertEquals("3 operator cycle", 3, vc.invalidCycles.get(0).size());
    assertTrue("operator2", vc.invalidCycles.get(0).contains(dag.getMeta(operator2)));
    assertTrue("operator3", vc.invalidCycles.get(0).contains(dag.getMeta(operator3)));
    assertTrue("operator4", vc.invalidCycles.get(0).contains(dag.getMeta(operator4)));
    try {
        dag.validate();
        fail("validation should fail");
    } catch (ValidationException e) {
    // expected
    }
}

Example 17 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanTest method testAffinityRulesDagValidation.

@Test
public void testAffinityRulesDagValidation() {
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1).setLocality(Locality.THREAD_LOCAL);
    StreamMeta stream2 = dag.addStream("stream2", o2.outport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O3");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    // Locality conflicts with affinity rules case:
    AffinityRule rule2 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule2);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O2 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change Stream2 locality to Node to check if validation passes
    stream2.setLocality(Locality.RACK_LOCAL);
    dag.validate();
    // Add anti-affinity rule conflicting with rule1
    AffinityRule rule3 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O3");
    rules.add(rule3);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O1 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change rule1 to Rack local to see if dag validation passes
    rules.clear();
    rule1.setLocality(Locality.RACK_LOCAL);
    rules.add(rule1);
    rules.add(rule2);
    rules.add(rule3);
    dag.validate();
    // Add conflicting rules and set relaxLocality for one rule
    AffinityRule rule4 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, true, "O1", "O2");
    rules.add(rule4);
    dag.validate();
    // Set conflicting host locality and check if it fails validation
    rules.clear();
    AffinityRule rule = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host1) conflict with anti-affinity rules", e.getMessage());
    }
    // Set conflicting affinity and different host locality for node-local
    // operators
    rules.clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host2) conflicts with affinity rules", e.getMessage());
    }
    // Check affinity Thread local validation for non-connected operators
    dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET).getAffinityRules().clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.THREAD_LOCAL, false, "O1", "O3");
    rules.add(rule);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Affinity rule specified THREAD_LOCAL affinity for operators O1 & O3 which are not connected by stream", e.getMessage());
    }
    // Check indirect conflict
    dag = new LogicalPlan();
    o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    o2 = dag.addOperator("O2", new GenericTestOperator());
    o3 = dag.addOperator("O3", new GenericTestOperator());
    GenericTestOperator o4 = dag.addOperator("O4", new GenericTestOperator());
    GenericTestOperator o5 = dag.addOperator("O5", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1, o3.inport1).setLocality(Locality.NODE_LOCAL);
    dag.addStream("stream2", o3.outport1, o4.inport1);
    dag.addStream("stream3", o2.outport1, o5.inport1);
    rules.clear();
    // O3 and O5 cannot have NODE_LOCAL anti-affinity now, since they already have NODE_LOCAL affinity
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O5"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O3", "O5"));
    ruleSet = new AffinityRulesSet();
    ruleSet.setAffinityRules(rules);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    try {
        dag.validate();
        Assert.fail("dag validation should fail due to conflicting affinity rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O3 & O5 conflicts with affinity rules or Stream locality", e.getMessage());
    }
}

Example 18 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanSerializer method convertToProperties.

public static PropertiesConfiguration convertToProperties(LogicalPlan dag) {
    PropertiesConfiguration props = new PropertiesConfiguration();
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    for (OperatorMeta operatorMeta : allOperators) {
        String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
        Operator operator = operatorMeta.getOperator();
        props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
        BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
        @SuppressWarnings("rawtypes") Iterator entryIterator = operatorProperties.entryIterator();
        while (entryIterator.hasNext()) {
            try {
                @SuppressWarnings("unchecked") Map.Entry<String, Object> entry = (Map.Entry<String, Object>) entryIterator.next();
                if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
                    props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
                }
            } catch (Exception ex) {
                LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
            }
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
        OutputPortMeta source = streamMeta.getSource();
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
        String sinksValue = "";
        for (InputPortMeta sink : sinks) {
            if (!sinksValue.isEmpty()) {
                sinksValue += ",";
            }
            sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
        }
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
        if (streamMeta.getLocality() != null) {
            props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
        }
    }
    return props;
}

Example 19 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanSerializer method convertToMap.

/**
   *
   * @param dag
   * @return
   */
public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules) {
    HashMap<String, Object> result = new HashMap<>();
    ArrayList<Object> operatorArray = new ArrayList<>();
    ArrayList<Object> streamMap = new ArrayList<>();
    //result.put("applicationName", appConfig.getName());
    result.put("operators", operatorArray);
    result.put("streams", streamMap);
    //LogicalPlan dag = StramAppLauncher.prepareDAG(appConfig, StreamingApplication.LAUNCHMODE_YARN);
    //
    // should we put the DAGContext info here?
    Map<String, Object> dagAttrs = new HashMap<>();
    for (Map.Entry<Attribute<Object>, Object> e : Attribute.AttributeMap.AttributeInitializer.getAllAttributes(dag, Context.DAGContext.class).entrySet()) {
        dagAttrs.put(e.getKey().getSimpleName(), e.getValue());
    }
    result.put("attributes", dagAttrs);
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    ObjectMapper propertyObjectMapper = new ObjectMapper();
    propertyObjectMapper.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, true);
    propertyObjectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
    StdTypeResolverBuilder typer = new PropertyTypeResolverBuilder();
    typer.init(JsonTypeInfo.Id.CLASS, null);
    typer = typer.inclusion(JsonTypeInfo.As.PROPERTY);
    propertyObjectMapper.setDefaultTyping(typer);
    for (OperatorMeta operatorMeta : allOperators) {
        HashMap<String, Object> operatorDetailMap = new HashMap<>();
        ArrayList<Map<String, Object>> portList = new ArrayList<>();
        Map<String, Object> attributeMap = new HashMap<>();
        String operatorName = operatorMeta.getName();
        operatorArray.add(operatorDetailMap);
        operatorDetailMap.put("name", operatorName);
        operatorDetailMap.put("ports", portList);
        operatorDetailMap.put("class", operatorMeta.getOperator().getClass().getName());
        operatorDetailMap.put("attributes", attributeMap);
        Map<Attribute<Object>, Object> rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(operatorMeta, Context.OperatorContext.class);
        for (Map.Entry<Attribute<Object>, Object> entry : rawAttributes.entrySet()) {
            attributeMap.put(entry.getKey().getSimpleName(), entry.getValue());
        }
        ObjectMapperString str;
        try {
            str = new ObjectMapperString(propertyObjectMapper.writeValueAsString(operatorMeta.getOperator()));
        } catch (Throwable ex) {
            LOG.error("Got exception when trying to get properties for operator {}", operatorMeta.getName(), ex);
            str = null;
        }
        operatorDetailMap.put("properties", str);
        Operators.PortMappingDescriptor pmd = new Operators.PortMappingDescriptor();
        Operators.describe(operatorMeta.getOperator(), pmd);
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : pmd.inputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            InputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "input");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : pmd.outputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            OutputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "output");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        HashMap<String, Object> streamDetailMap = new HashMap<>();
        String streamName = streamMeta.getName();
        streamMap.add(streamDetailMap);
        String sourcePortName = streamMeta.getSource().getPortName();
        OperatorMeta operatorMeta = streamMeta.getSource().getOperatorMeta();
        HashMap<String, Object> sourcePortDetailMap = new HashMap<>();
        sourcePortDetailMap.put("operatorName", operatorMeta.getName());
        sourcePortDetailMap.put("portName", sourcePortName);
        streamDetailMap.put("name", streamName);
        streamDetailMap.put("source", sourcePortDetailMap);
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        ArrayList<HashMap<String, Object>> sinkPortList = new ArrayList<>();
        for (InputPortMeta sinkPort : sinks) {
            HashMap<String, Object> sinkPortDetailMap = new HashMap<>();
            sinkPortDetailMap.put("operatorName", sinkPort.getOperatorMeta().getName());
            sinkPortDetailMap.put("portName", sinkPort.getPortName());
            sinkPortList.add(sinkPortDetailMap);
        }
        streamDetailMap.put("sinks", sinkPortList);
        if (streamMeta.getLocality() != null) {
            streamDetailMap.put("locality", streamMeta.getLocality().name());
        }
    }
    if (includeModules) {
        ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
        result.put("modules", modulesArray);
        for (LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
            modulesArray.add(getLogicalModuleDetails(dag, meta));
        }
    }
    return result;
}

Example 20 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class OiOEndWindowTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
    FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
    SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());
    /*
     * This tests make sure that even if the application_window_count is different the endWindow() is called for
     * end_stream
     */
    lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);
    stream1.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}