Examples with OutputPort com.datatorrent.api.Operator.OutputPort used on opensource projects

Example 1 with OutputPort

use of com.datatorrent.api.Operator.OutputPort in project beam by apache.

the class ParDoTranslator method translate.

@Override
public void translate(ParDo.MultiOutput<InputT, OutputT> transform, TranslationContext context) {
    DoFn<InputT, OutputT> doFn = transform.getFn();
    DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass());
    if (signature.processElement().isSplittable()) {
        throw new UnsupportedOperationException(String.format("%s does not support splittable DoFn: %s", ApexRunner.class.getSimpleName(), doFn));
    }
    if (signature.stateDeclarations().size() > 0) {
        throw new UnsupportedOperationException(String.format("Found %s annotations on %s, but %s cannot yet be used with state in the %s.", DoFn.StateId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), ApexRunner.class.getSimpleName()));
    }
    if (signature.timerDeclarations().size() > 0) {
        throw new UnsupportedOperationException(String.format("Found %s annotations on %s, but %s cannot yet be used with timers in the %s.", DoFn.TimerId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), ApexRunner.class.getSimpleName()));
    }
    Map<TupleTag<?>, PValue> outputs = context.getOutputs();
    PCollection<InputT> input = context.getInput();
    List<PCollectionView<?>> sideInputs = transform.getSideInputs();
    Coder<InputT> inputCoder = input.getCoder();
    WindowedValueCoder<InputT> wvInputCoder = FullWindowedValueCoder.of(inputCoder, input.getWindowingStrategy().getWindowFn().windowCoder());
    ApexParDoOperator<InputT, OutputT> operator = new ApexParDoOperator<>(context.getPipelineOptions(), doFn, transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), input.getWindowingStrategy(), sideInputs, wvInputCoder, context.getStateBackend());
    Map<PCollection<?>, OutputPort<?>> ports = Maps.newHashMapWithExpectedSize(outputs.size());
    for (Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
        checkArgument(output.getValue() instanceof PCollection, "%s %s outputs non-PCollection %s of type %s", ParDo.MultiOutput.class.getSimpleName(), context.getFullName(), output.getValue(), output.getValue().getClass().getSimpleName());
        PCollection<?> pc = (PCollection<?>) output.getValue();
        if (output.getKey().equals(transform.getMainOutputTag())) {
            ports.put(pc, operator.output);
        } else {
            int portIndex = 0;
            for (TupleTag<?> tag : transform.getAdditionalOutputTags().getAll()) {
                if (tag.equals(output.getKey())) {
                    ports.put(pc, operator.additionalOutputPorts[portIndex]);
                    break;
                }
                portIndex++;
            }
        }
    }
    context.addOperator(operator, ports);
    context.addStream(context.getInput(), operator.input);
    if (!sideInputs.isEmpty()) {
        addSideInputs(operator.sideInput1, sideInputs, context);
    }
}

Example 2 with OutputPort

use of com.datatorrent.api.Operator.OutputPort in project apex-core by apache.

the class StreamingContainer method setupNode.

private void setupNode(OperatorDeployInfo ndi) {
    failedNodes.remove(ndi.id);
    final Node<?> node = nodes.get(ndi.id);
    node.setup(node.context);
    /* setup context for all the input ports */
    LinkedHashMap<String, PortContextPair<InputPort<?>>> inputPorts = node.getPortMappingDescriptor().inputPorts;
    LinkedHashMap<String, PortContextPair<InputPort<?>>> newInputPorts = new LinkedHashMap<>(inputPorts.size());
    for (OperatorDeployInfo.InputDeployInfo idi : ndi.inputs) {
        InputPort<?> port = inputPorts.get(idi.portName).component;
        PortContext context = new PortContext(idi.contextAttributes, node.context);
        newInputPorts.put(idi.portName, new PortContextPair<InputPort<?>>(port, context));
        port.setup(context);
    }
    inputPorts.putAll(newInputPorts);
    /* setup context for all the output ports */
    LinkedHashMap<String, PortContextPair<OutputPort<?>>> outputPorts = node.getPortMappingDescriptor().outputPorts;
    LinkedHashMap<String, PortContextPair<OutputPort<?>>> newOutputPorts = new LinkedHashMap<>(outputPorts.size());
    for (OperatorDeployInfo.OutputDeployInfo odi : ndi.outputs) {
        OutputPort<?> port = outputPorts.get(odi.portName).component;
        PortContext context = new PortContext(odi.contextAttributes, node.context);
        newOutputPorts.put(odi.portName, new PortContextPair<OutputPort<?>>(port, context));
        port.setup(context);
    }
    outputPorts.putAll(newOutputPorts);
    logger.debug("activating {} in container {}", node, containerId);
    /* This introduces need for synchronization on processNodeRequest which was solved by adding deleted field in StramToNodeRequest  */
    processNodeRequests(false);
    node.activate();
    eventBus.publish(new NodeActivationEvent(node));
}

Example 3 with OutputPort

use of com.datatorrent.api.Operator.OutputPort in project apex-core by apache.

the class TupleRecorderCollection method startRecording.

private void startRecording(String id, final Node<?> node, int operatorId, final String portName, long numWindows) {
    PortMappingDescriptor descriptor = node.getPortMappingDescriptor();
    OperatorIdPortNamePair operatorIdPortNamePair = new OperatorIdPortNamePair(operatorId, portName);
    // check any recording conflict
    boolean conflict = false;
    if (containsKey(new OperatorIdPortNamePair(operatorId, null))) {
        conflict = true;
    } else if (portName == null) {
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            if (containsKey(new OperatorIdPortNamePair(operatorId, entry.getKey()))) {
                conflict = true;
                break;
            }
        }
    } else {
        if (containsKey(operatorIdPortNamePair)) {
            conflict = true;
        }
    }
    if (!conflict) {
        logger.debug("Executing start recording request for {}", operatorIdPortNamePair);
        if (wsClient != null) {
            try {
                wsClient.openConnection();
            } catch (Exception e) {
                logger.warn("Cannot establish websocket connection to uri {}", wsClient.getUri(), e);
            }
        }
        TupleRecorder tupleRecorder = new TupleRecorder(id, appId);
        tupleRecorder.setWebSocketClient(wsClient);
        HashMap<String, Sink<Object>> sinkMap = new HashMap<>();
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : descriptor.inputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (entry.getValue().context != null && (portName == null || entry.getKey().equals(portName))) {
                logger.debug("Adding recorder sink to input port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addInputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : descriptor.outputPorts.entrySet()) {
            String streamId = getDeclaredStreamId(operatorId, entry.getKey());
            if (streamId == null) {
                streamId = portName + "_implicit_stream";
            }
            if (portName == null || entry.getKey().equals(portName)) {
                logger.debug("Adding recorder sink to output port {}, stream {}", entry.getKey(), streamId);
                tupleRecorder.addOutputPortInfo(entry.getKey(), streamId);
                sinkMap.put(entry.getKey(), tupleRecorder.newSink(entry.getKey()));
            }
        }
        if (!sinkMap.isEmpty()) {
            logger.debug("Started recording on {} through {}", operatorIdPortNamePair, System.identityHashCode(this));
            String basePath = appPath + "/recordings/" + operatorId + "/" + tupleRecorder.getId();
            tupleRecorder.getStorage().setBasePath(basePath);
            tupleRecorder.getStorage().setBytesPerPartFile(tupleRecordingPartFileSize);
            tupleRecorder.getStorage().setMillisPerPartFile(tupleRecordingPartFileTimeMillis);
            node.addSinks(sinkMap);
            tupleRecorder.setup(node.getOperator(), codecs);
            put(operatorIdPortNamePair, tupleRecorder);
            if (numWindows > 0) {
                tupleRecorder.setNumWindows(numWindows, new Runnable() {

                    @Override
                    public void run() {
                        node.context.request(new OperatorRequest() {

                            @Override
                            public StatsListener.OperatorResponse execute(Operator operator, int operatorId, long windowId) throws IOException {
                                stopRecording(node, operatorId, portName);
                                return null;
                            }
                        });
                    }
                });
            }
        } else {
            logger.warn("Tuple recording request ignored because operator is not connected on the specified port.");
        }
    } else {
        logger.error("Operator id {} is already being recorded.", operatorId);
    }
}

Example 4 with OutputPort

use of com.datatorrent.api.Operator.OutputPort in project apex-core by apache.

the class StreamingContainerManager method getPortAttributes.

public Attribute.AttributeMap getPortAttributes(String operatorId, String portName) {
    OperatorMeta logicalOperator = plan.getLogicalPlan().getOperatorMeta(operatorId);
    if (logicalOperator == null) {
        throw new IllegalArgumentException("Invalid operatorId " + operatorId);
    }
    Operators.PortMappingDescriptor portMap = new Operators.PortMappingDescriptor();
    Operators.describe(logicalOperator.getOperator(), portMap);
    PortContextPair<InputPort<?>> inputPort = portMap.inputPorts.get(portName);
    if (inputPort != null) {
        InputPortMeta portMeta = logicalOperator.getMeta(inputPort.component);
        try {
            return portMeta.getAttributes().clone();
        } catch (CloneNotSupportedException ex) {
            throw new RuntimeException("Cannot clone port attributes", ex);
        }
    } else {
        PortContextPair<OutputPort<?>> outputPort = portMap.outputPorts.get(portName);
        if (outputPort != null) {
            OutputPortMeta portMeta = logicalOperator.getMeta(outputPort.component);
            try {
                return portMeta.getAttributes().clone();
            } catch (CloneNotSupportedException ex) {
                throw new RuntimeException("Cannot clone port attributes", ex);
            }
        }
        throw new IllegalArgumentException("Invalid port name " + portName);
    }
}

Example 5 with OutputPort

use of com.datatorrent.api.Operator.OutputPort in project apex-core by apache.

the class Operators method describe.

public static void describe(GenericOperator operator, OperatorDescriptor descriptor) {
    for (Class<?> c = operator.getClass(); c != Object.class; c = c.getSuperclass()) {
        Field[] fields = c.getDeclaredFields();
        for (Field field : fields) {
            field.setAccessible(true);
            InputPortFieldAnnotation inputAnnotation = field.getAnnotation(InputPortFieldAnnotation.class);
            OutputPortFieldAnnotation outputAnnotation = field.getAnnotation(OutputPortFieldAnnotation.class);
            AppData.QueryPort adqAnnotation = field.getAnnotation(AppData.QueryPort.class);
            AppData.ResultPort adrAnnotation = field.getAnnotation(AppData.ResultPort.class);
            try {
                Object portObject = field.get(operator);
                if (portObject instanceof InputPort) {
                    descriptor.addInputPort((Operator.InputPort<?>) portObject, field, inputAnnotation, adqAnnotation);
                } else {
                    if (inputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + InputPort.class.getName() + ": " + field);
                    }
                }
                if (portObject instanceof OutputPort) {
                    descriptor.addOutputPort((Operator.OutputPort<?>) portObject, field, outputAnnotation, adrAnnotation);
                } else {
                    if (outputAnnotation != null) {
                        throw new IllegalArgumentException("port is not of type " + OutputPort.class.getName() + ": " + field);
                    }
                }
            } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
            }
        }
    }
}