Examples with GenericOperator com.datatorrent.api.DAG.GenericOperator used on opensource projects

Example 1 with GenericOperator

use of com.datatorrent.api.DAG.GenericOperator in project apex-core by apache.

the class LogicalPlanConfiguration method populateDAG.

/**
   * Populate the logical plan structure from properties.
   * @param dag
   */
public void populateDAG(LogicalPlan dag) {
    Configuration pconf = new Configuration(conf);
    for (final String propertyName : this.properties.stringPropertyNames()) {
        String propertyValue = this.properties.getProperty(propertyName);
        pconf.setIfUnset(propertyName, propertyValue);
    }
    AppConf appConf = this.stramConf.getChild(WILDCARD, StramElement.APPLICATION);
    if (appConf == null) {
        LOG.warn("Application configuration not found. Probably an empty app.");
        return;
    }
    Map<String, OperatorConf> operators = appConf.getChildren(StramElement.OPERATOR);
    Map<OperatorConf, GenericOperator> nodeMap = Maps.newHashMapWithExpectedSize(operators.size());
    // add all operators first
    for (Map.Entry<String, OperatorConf> nodeConfEntry : operators.entrySet()) {
        OperatorConf nodeConf = nodeConfEntry.getValue();
        if (!WILDCARD.equals(nodeConf.id)) {
            Class<? extends GenericOperator> nodeClass = StramUtils.classForName(nodeConf.getClassNameReqd(), GenericOperator.class);
            String optJson = nodeConf.getProperties().get(nodeClass.getName());
            GenericOperator operator = null;
            try {
                if (optJson != null) {
                    // if there is a special key which is the class name, it means the operator is serialized in json format
                    ObjectMapper mapper = ObjectMapperFactory.getOperatorValueDeserializer();
                    operator = mapper.readValue("{\"" + nodeClass.getName() + "\":" + optJson + "}", nodeClass);
                    addOperator(dag, nodeConfEntry.getKey(), operator);
                } else {
                    operator = addOperator(dag, nodeConfEntry.getKey(), nodeClass);
                }
                setOperatorProperties(operator, nodeConf.getProperties());
            } catch (IOException e) {
                throw new IllegalArgumentException("Error setting operator properties " + e.getMessage(), e);
            }
            nodeMap.put(nodeConf, operator);
        }
    }
    Map<String, StreamConf> streams = appConf.getChildren(StramElement.STREAM);
    // wire operators
    for (Map.Entry<String, StreamConf> streamConfEntry : streams.entrySet()) {
        StreamConf streamConf = streamConfEntry.getValue();
        DAG.StreamMeta sd = dag.addStream(streamConfEntry.getKey());
        sd.setLocality(streamConf.getLocality());
        String schemaClassName = streamConf.properties.getProperty(STREAM_SCHEMA);
        Class<?> schemaClass = null;
        if (schemaClassName != null) {
            schemaClass = StramUtils.classForName(schemaClassName, Object.class);
        }
        if (streamConf.sourceNode != null) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : streamConf.sourceNode.outputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator sourceDecl = nodeMap.get(streamConf.sourceNode);
            Operators.PortMappingDescriptor sourcePortMap = new Operators.PortMappingDescriptor();
            Operators.describe(sourceDecl, sourcePortMap);
            sd.setSource(sourcePortMap.outputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setOutputPortAttribute(sourcePortMap.outputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
        for (OperatorConf targetNode : streamConf.targetNodes) {
            String portName = null;
            for (Map.Entry<String, StreamConf> e : targetNode.inputs.entrySet()) {
                if (e.getValue() == streamConf) {
                    portName = e.getKey();
                }
            }
            GenericOperator targetDecl = nodeMap.get(targetNode);
            Operators.PortMappingDescriptor targetPortMap = new Operators.PortMappingDescriptor();
            Operators.describe(targetDecl, targetPortMap);
            sd.addSink(targetPortMap.inputPorts.get(portName).component);
            if (schemaClass != null) {
                dag.setInputPortAttribute(targetPortMap.inputPorts.get(portName).component, PortContext.TUPLE_CLASS, schemaClass);
            }
        }
    }
}

Example 2 with GenericOperator

use of com.datatorrent.api.DAG.GenericOperator in project apex-core by apache.

the class Operators method describe.

public static void describe(GenericOperator operator, OperatorDescriptor descriptor) {
    for (Class<?> c = operator.getClass(); c != Object.class; c = c.getSuperclass()) {
        Field[] fields = c.getDeclaredFields();
        for (Field field : fields) {
            field.setAccessible(true);
            InputPortFieldAnnotation inputAnnotation = field.getAnnotation(InputPortFieldAnnotation.class);
            OutputPortFieldAnnotation outputAnnotation = field.getAnnotation(OutputPortFieldAnnotation.class);
            AppData.QueryPort adqAnnotation = field.getAnnotation(AppData.QueryPort.class);
            AppData.ResultPort adrAnnotation = field.getAnnotation(AppData.ResultPort.class);
            try {
                Object portObject = field.get(operator);
                if (portObject instanceof InputPort) {
                    descriptor.addInputPort((Operator.InputPort<?>) portObject, field, inputAnnotation, adqAnnotation);
                } else {
                    if (inputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + InputPort.class.getName() + ": " + field);
                    }
                }
                if (portObject instanceof OutputPort) {
                    descriptor.addOutputPort((Operator.OutputPort<?>) portObject, field, outputAnnotation, adrAnnotation);
                } else {
                    if (outputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + OutputPort.class.getName() + ": " + field);
                    }
                }
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    }
}

Example 3 with GenericOperator

use of com.datatorrent.api.DAG.GenericOperator in project apex-core by apache.

the class OperatorDiscoverer method describeOperator.

public JSONObject describeOperator(String clazz) throws Exception {
    TypeGraphVertex tgv = typeGraph.getTypeGraphVertex(clazz);
    if (tgv.isInstantiable()) {
        JSONObject response = new JSONObject();
        JSONArray inputPorts = new JSONArray();
        JSONArray outputPorts = new JSONArray();
        // Get properties from ASM
        JSONObject operatorDescriptor = describeClassByASM(clazz);
        JSONArray properties = operatorDescriptor.getJSONArray("properties");
        properties = enrichProperties(clazz, properties);
        JSONArray portTypeInfo = operatorDescriptor.getJSONArray("portTypeInfo");
        List<CompactFieldNode> inputPortfields = typeGraph.getAllInputPorts(clazz);
        List<CompactFieldNode> outputPortfields = typeGraph.getAllOutputPorts(clazz);
        try {
            for (CompactFieldNode field : inputPortfields) {
                JSONObject inputPort = setFieldAttributes(clazz, field);
                if (!inputPort.has("optional")) {
                    // input port that is not annotated is default to be not optional
                    inputPort.put("optional", false);
                }
                if (!inputPort.has(SCHEMA_REQUIRED_KEY)) {
                    inputPort.put(SCHEMA_REQUIRED_KEY, false);
                }
                inputPorts.put(inputPort);
            }
            for (CompactFieldNode field : outputPortfields) {
                JSONObject outputPort = setFieldAttributes(clazz, field);
                if (!outputPort.has("optional")) {
                    // output port that is not annotated is default to be optional
                    outputPort.put("optional", true);
                }
                if (!outputPort.has("error")) {
                    outputPort.put("error", false);
                }
                if (!outputPort.has(SCHEMA_REQUIRED_KEY)) {
                    outputPort.put(SCHEMA_REQUIRED_KEY, false);
                }
                outputPorts.put(outputPort);
            }
            response.put("name", clazz);
            response.put("properties", properties);
            response.put(PORT_TYPE_INFO_KEY, portTypeInfo);
            response.put("inputPorts", inputPorts);
            response.put("outputPorts", outputPorts);
            String type = null;
            Class<?> genericOperator = classLoader.loadClass(clazz);
            if (Module.class.isAssignableFrom(genericOperator)) {
                type = GenericOperatorType.MODULE.getType();
            } else if (Operator.class.isAssignableFrom(genericOperator)) {
                type = GenericOperatorType.OPERATOR.getType();
            }
            if (type != null) {
                response.put("type", type);
            }
            OperatorClassInfo oci = classInfo.get(clazz);
            if (oci != null) {
                if (oci.comment != null) {
                    String[] descriptions;
                    // first look for a <p> tag
                    String keptPrefix = "<p>";
                    descriptions = oci.comment.split("<p>", 2);
                    if (descriptions.length == 0) {
                        keptPrefix = "";
                        // if no <p> tag, then look for a blank line
                        descriptions = oci.comment.split("\n\n", 2);
                    }
                    if (descriptions.length > 0) {
                        response.put("shortDesc", descriptions[0]);
                    }
                    if (descriptions.length > 1) {
                        response.put("longDesc", keptPrefix + descriptions[1]);
                    }
                }
                response.put("category", oci.tags.get("@category"));
                String displayName = oci.tags.get("@displayName");
                if (displayName == null) {
                    displayName = decamelizeClassName(ClassUtils.getShortClassName(clazz));
                }
                response.put("displayName", displayName);
                String tags = oci.tags.get("@tags");
                if (tags != null) {
                    JSONArray tagArray = new JSONArray();
                    for (String tag : StringUtils.split(tags, ',')) {
                        tagArray.put(tag.trim().toLowerCase());
                    }
                    response.put("tags", tagArray);
                }
                String doclink = oci.tags.get("@doclink");
                if (doclink != null) {
                    response.put("doclink", doclink + "?" + getDocName(clazz));
                } else if (clazz.startsWith("com.datatorrent.lib.") || clazz.startsWith("com.datatorrent.contrib.")) {
                    response.put("doclink", DT_OPERATOR_DOCLINK_PREFIX + "?" + getDocName(clazz));
                }
            }
        } catch (JSONException ex) {
            throw new RuntimeException(ex);
        }
        return response;
    } else {
        throw new UnsupportedOperationException();
    }
}

Example 4 with GenericOperator

use of com.datatorrent.api.DAG.GenericOperator in project apex-core by apache.

the class OperatorDiscoverer method addDefaultValue.

@SuppressWarnings("unchecked")
public void addDefaultValue(String className, JSONObject oper) throws Exception {
    ObjectMapper defaultValueMapper = ObjectMapperFactory.getOperatorValueSerializer();
    Class<? extends GenericOperator> clazz = (Class<? extends GenericOperator>) classLoader.loadClass(className);
    if (clazz != null) {
        GenericOperator operIns = clazz.newInstance();
        String s = defaultValueMapper.writeValueAsString(operIns);
        oper.put("defaultValue", new JSONObject(s).get(className));
    }
}