Examples with PortContextPair com.datatorrent.stram.plan.logical.Operators.PortContextPair used on opensource projects

Example 1 with PortContextPair

use of com.datatorrent.stram.plan.logical.Operators.PortContextPair in project apex-core by apache.

the class StreamingContainer method setupNode.

private void setupNode(OperatorDeployInfo ndi) {
    failedNodes.remove(ndi.id);
    final Node<?> node = nodes.get(ndi.id);
    node.setup(node.context);
    /* setup context for all the input ports */
    LinkedHashMap<String, PortContextPair<InputPort<?>>> inputPorts = node.getPortMappingDescriptor().inputPorts;
    LinkedHashMap<String, PortContextPair<InputPort<?>>> newInputPorts = new LinkedHashMap<>(inputPorts.size());
    for (OperatorDeployInfo.InputDeployInfo idi : ndi.inputs) {
        InputPort<?> port = inputPorts.get(idi.portName).component;
        PortContext context = new PortContext(idi.contextAttributes, node.context);
        newInputPorts.put(idi.portName, new PortContextPair<InputPort<?>>(port, context));
        port.setup(context);
    }
    inputPorts.putAll(newInputPorts);
    /* setup context for all the output ports */
    LinkedHashMap<String, PortContextPair<OutputPort<?>>> outputPorts = node.getPortMappingDescriptor().outputPorts;
    LinkedHashMap<String, PortContextPair<OutputPort<?>>> newOutputPorts = new LinkedHashMap<>(outputPorts.size());
    for (OperatorDeployInfo.OutputDeployInfo odi : ndi.outputs) {
        OutputPort<?> port = outputPorts.get(odi.portName).component;
        PortContext context = new PortContext(odi.contextAttributes, node.context);
        newOutputPorts.put(odi.portName, new PortContextPair<OutputPort<?>>(port, context));
        port.setup(context);
    }
    outputPorts.putAll(newOutputPorts);
    logger.debug("activating {} in container {}", node, containerId);
    /* This introduces need for synchronization on processNodeRequest which was solved by adding deleted field in StramToNodeRequest  */
    processNodeRequests(false);
    node.activate();
    eventBus.publish(new NodeActivationEvent(node));
}

Example 2 with PortContextPair

use of com.datatorrent.stram.plan.logical.Operators.PortContextPair in project apex-core by apache.

the class TupleRecorderCollection method startRecording.

private void startRecording(String id, final Node<?> node, int operatorId, final String portName, long numWindows) {
    PortMappingDescriptor descriptor = node.getPortMappingDescriptor();
    OperatorIdPortNamePair operatorIdPortNamePair = new OperatorIdPortNamePair(operatorId, portName);
    // check any recording conflict
    boolean conflict = false;
    if (containsKey(new OperatorIdPortNamePair(operatorId, null))) {
        conflict = true;
    } else if (portName == null) {
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
    } else {
        if (containsKey(operatorIdPortNamePair)) {
            conflict = true;
        }
    }
    if (!conflict) {
        logger.debug("Executing start recording request for {}", operatorIdPortNamePair);
        if (wsClient != null) {
            try {
                wsClient.openConnection();
            } catch (Exception e) {
                logger.warn("Cannot establish websocket connection to uri {}", wsClient.getUri(), e);
            }
        }
        TupleRecorder tupleRecorder = new TupleRecorder(id, appId);
        tupleRecorder.setWebSocketClient(wsClient);
        HashMap<String, Sink<Object>> sinkMap = new HashMap<>();
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (entry.getValue().context != null && (portName == null || entry.getKey().equals(portName))) {
                logger.debug("Adding recorder sink to input port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addInputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (portName == null || entry.getKey().equals(portName)) {
                logger.debug("Adding recorder sink to output port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addOutputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        if (!sinkMap.isEmpty()) {
            logger.debug("Started recording on {} through {}", operatorIdPortNamePair, System.identityHashCode(this));
            String basePath = appPath + "/recordings/" + operatorId + "/" + tupleRecorder.getId();
            tupleRecorder.getStorage().setBasePath(basePath);
            tupleRecorder.getStorage().setBytesPerPartFile(tupleRecordingPartFileSize);
            tupleRecorder.getStorage().setMillisPerPartFile(tupleRecordingPartFileTimeMillis);
            node.addSinks(sinkMap);
            tupleRecorder.setup(node.getOperator(), codecs);
            put(operatorIdPortNamePair, tupleRecorder);
            if (numWindows > 0) {
                tupleRecorder.setNumWindows(numWindows, new Runnable() {

                    @Override
                    public void run() {
                        node.context.request(new OperatorRequest() {

                            @Override
                            public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException {
                                stopRecording(node, operatorId, portName);
                                return null;
                            }
                        });
                    }
                });
            }
        } else {
            logger.warn("Tuple recording request ignored because operator is not connected on the specified port.");
        }
    } else {
        logger.error("Operator id {} is already being recorded.", operatorId);
    }
}

Example 3 with PortContextPair

use of com.datatorrent.stram.plan.logical.Operators.PortContextPair in project apex-core by apache.

the class LogicalPlanSerializer method convertToMap.

/**
 * @param dag
 * @return
 */
public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules) {
    HashMap<String, Object> result = new HashMap<>();
    ArrayList<Object> operatorArray = new ArrayList<>();
    ArrayList<Object> streamMap = new ArrayList<>();
    // result.put("applicationName", appConfig.getName());
    result.put("operators", operatorArray);
    result.put("streams", streamMap);
    // LogicalPlan dag = StramAppLauncher.prepareDAG(appConfig, StreamingApplication.LAUNCHMODE_YARN);
    // 
    // should we put the DAGContext info here?
    Map<String, Object> dagAttrs = new HashMap<>();
    for (Map.Entry<Attribute<Object>, Object> e : Attribute.AttributeMap.AttributeInitializer.getAllAttributes(dag, Context.DAGContext.class).entrySet()) {
        dagAttrs.put(e.getKey().getSimpleName(), e.getValue());
    }
    result.put("attributes", dagAttrs);
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    ObjectMapper propertyObjectMapper = new ObjectMapper();
    propertyObjectMapper.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, true);
    propertyObjectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
    StdTypeResolverBuilder typer = new PropertyTypeResolverBuilder();
    typer.init(JsonTypeInfo.Id.CLASS, null);
    typer = typer.inclusion(JsonTypeInfo.As.PROPERTY);
    propertyObjectMapper.setDefaultTyping(typer);
    for (OperatorMeta operatorMeta : allOperators) {
        HashMap<String, Object> operatorDetailMap = new HashMap<>();
        ArrayList<Map<String, Object>> portList = new ArrayList<>();
        Map<String, Object> attributeMap = new HashMap<>();
        String operatorName = operatorMeta.getName();
        operatorArray.add(operatorDetailMap);
        operatorDetailMap.put("name", operatorName);
        operatorDetailMap.put("ports", portList);
        operatorDetailMap.put("class", operatorMeta.getOperator().getClass().getName());
        operatorDetailMap.put("attributes", attributeMap);
        Map<Attribute<Object>, Object> rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(operatorMeta, Context.OperatorContext.class);
        for (Map.Entry<Attribute<Object>, Object> entry : rawAttributes.entrySet()) {
            attributeMap.put(entry.getKey().getSimpleName(), entry.getValue());
        }
        ObjectMapperString str;
        try {
            str = new ObjectMapperString(propertyObjectMapper.writeValueAsString(operatorMeta.getOperator()));
        } catch (Throwable ex) {
            LOG.error("Got exception when trying to get properties for operator {}", operatorMeta.getName(), ex);
            str = null;
        }
        operatorDetailMap.put("properties", str);
        Operators.PortMappingDescriptor pmd = new Operators.PortMappingDescriptor();
        Operators.describe(operatorMeta.getOperator(), pmd);
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : pmd.inputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            InputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "input");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : pmd.outputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            OutputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "output");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        HashMap<String, Object> streamDetailMap = new HashMap<>();
        String streamName = streamMeta.getName();
        streamMap.add(streamDetailMap);
        String sourcePortName = streamMeta.getSource().getPortName();
        OperatorMeta operatorMeta = streamMeta.getSource().getOperatorMeta();
        HashMap<String, Object> sourcePortDetailMap = new HashMap<>();
        sourcePortDetailMap.put("operatorName", operatorMeta.getName());
        sourcePortDetailMap.put("portName", sourcePortName);
        streamDetailMap.put("name", streamName);
        streamDetailMap.put("source", sourcePortDetailMap);
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        ArrayList<HashMap<String, Object>> sinkPortList = new ArrayList<>();
        for (InputPortMeta sinkPort : sinks) {
            HashMap<String, Object> sinkPortDetailMap = new HashMap<>();
            sinkPortDetailMap.put("operatorName", sinkPort.getOperatorMeta().getName());
            sinkPortDetailMap.put("portName", sinkPort.getPortName());
            sinkPortList.add(sinkPortDetailMap);
        }
        streamDetailMap.put("sinks", sinkPortList);
        if (streamMeta.getLocality() != null) {
            streamDetailMap.put("locality", streamMeta.getLocality().name());
        }
    }
    if (includeModules) {
        ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
        result.put("modules", modulesArray);
        for (LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
            modulesArray.add(getLogicalModuleDetails(dag, meta));
        }
    }
    return result;
}