Examples with RecordWriterOutput org.apache.flink.streaming.runtime.io.RecordWriterOutput used on opensource projects

Example 1 with RecordWriterOutput

use of org.apache.flink.streaming.runtime.io.RecordWriterOutput in project flink by apache.

the class StreamIterationHead method run.

// ------------------------------------------------------------------------
@Override
protected void run() throws Exception {
    final String iterationId = getConfiguration().getIterationId();
    if (iterationId == null || iterationId.length() == 0) {
        throw new Exception("Missing iteration ID in the task configuration");
    }
    final String brokerID = createBrokerIdString(getEnvironment().getJobID(), iterationId, getEnvironment().getTaskInfo().getIndexOfThisSubtask());
    final long iterationWaitTime = getConfiguration().getIterationWaitTime();
    final boolean shouldWait = iterationWaitTime > 0;
    final BlockingQueue<StreamRecord<OUT>> dataChannel = new ArrayBlockingQueue<StreamRecord<OUT>>(1);
    // offer the queue for the tail
    BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
    LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);
    // do the work 
    try {
        @SuppressWarnings("unchecked") RecordWriterOutput<OUT>[] outputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();
        // If timestamps are enabled we make sure to remove cyclic watermark dependencies
        if (isSerializingTimestamps()) {
            for (RecordWriterOutput<OUT> output : outputs) {
                output.emitWatermark(new Watermark(Long.MAX_VALUE));
            }
        }
        while (running) {
            StreamRecord<OUT> nextRecord = shouldWait ? dataChannel.poll(iterationWaitTime, TimeUnit.MILLISECONDS) : dataChannel.take();
            if (nextRecord != null) {
                for (RecordWriterOutput<OUT> output : outputs) {
                    output.collect(nextRecord);
                }
            } else {
                // done
                break;
            }
        }
    } finally {
        // make sure that we remove the queue from the broker, to prevent a resource leak
        BlockingQueueBroker.INSTANCE.remove(brokerID);
        LOG.info("Iteration head {} removed feedback queue under {}", getName(), brokerID);
    }
}

Example 2 with RecordWriterOutput

use of org.apache.flink.streaming.runtime.io.RecordWriterOutput in project flink by apache.

the class StreamIterationHead method init.

// ------------------------------------------------------------------------
@SuppressWarnings("unchecked")
@Override
public void init() {
    // offer the queue for the tail
    BlockingQueueBroker.INSTANCE.handIn(brokerID, dataChannel);
    LOG.info("Iteration head {} added feedback queue under {}", getName(), brokerID);
    this.streamOutputs = (RecordWriterOutput<OUT>[]) getStreamOutputs();
    // If timestamps are enabled we make sure to remove cyclic watermark dependencies
    if (isSerializingTimestamps()) {
        for (RecordWriterOutput<OUT> output : streamOutputs) {
            output.emitWatermark(new Watermark(Long.MAX_VALUE));
        }
    }
}

Example 3 with RecordWriterOutput

use of org.apache.flink.streaming.runtime.io.RecordWriterOutput in project flink by apache.

the class OperatorChain method createStreamOutput.

private <T> RecordWriterOutput<T> createStreamOutput(StreamEdge edge, StreamConfig upStreamConfig, int outputIndex, Environment taskEnvironment, String taskName) {
    // OutputTag, return null if not sideOutput
    OutputTag sideOutputTag = edge.getOutputTag();
    TypeSerializer outSerializer = null;
    if (edge.getOutputTag() != null) {
        // side output
        outSerializer = upStreamConfig.getTypeSerializerSideOut(edge.getOutputTag(), taskEnvironment.getUserClassLoader());
    } else {
        // main output
        outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
    }
    @SuppressWarnings("unchecked") StreamPartitioner<T> outputPartitioner = (StreamPartitioner<T>) edge.getPartitioner();
    LOG.debug("Using partitioner {} for output {} of task ", outputPartitioner, outputIndex, taskName);
    ResultPartitionWriter bufferWriter = taskEnvironment.getWriter(outputIndex);
    // we initialize the partitioner here with the number of key groups (aka max. parallelism)
    if (outputPartitioner instanceof ConfigurableStreamPartitioner) {
        int numKeyGroups = bufferWriter.getNumTargetKeyGroups();
        if (0 < numKeyGroups) {
            ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups);
        }
    }
    StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout());
    output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup());
    return new RecordWriterOutput<>(output, outSerializer, sideOutputTag, this);
}

Example 4 with RecordWriterOutput

use of org.apache.flink.streaming.runtime.io.RecordWriterOutput in project flink by apache.

the class OperatorChain method createOutputCollector.

// ------------------------------------------------------------------------
//  initialization utilities
// ------------------------------------------------------------------------
private <T> Output<StreamRecord<T>> createOutputCollector(StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperator<?>> allOperators) {
    List<Tuple2<Output<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
    // create collectors for the network outputs
    for (StreamEdge outputEdge : operatorConfig.getNonChainedOutputs(userCodeClassloader)) {
        @SuppressWarnings("unchecked") RecordWriterOutput<T> output = (RecordWriterOutput<T>) streamOutputs.get(outputEdge);
        allOutputs.add(new Tuple2<Output<StreamRecord<T>>, StreamEdge>(output, outputEdge));
    }
    // Create collectors for the chained outputs
    for (StreamEdge outputEdge : operatorConfig.getChainedOutputs(userCodeClassloader)) {
        int outputId = outputEdge.getTargetId();
        StreamConfig chainedOpConfig = chainedConfigs.get(outputId);
        Output<StreamRecord<T>> output = createChainedOperator(containingTask, chainedOpConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperators, outputEdge.getOutputTag());
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    // if there are multiple outputs, or the outputs are directed, we need to
    // wrap them as one output
    List<OutputSelector<T>> selectors = operatorConfig.getOutputSelectors(userCodeClassloader);
    if (selectors == null || selectors.isEmpty()) {
        // simple path, no selector necessary
        if (allOutputs.size() == 1) {
            return allOutputs.get(0).f0;
        } else {
            // send to N outputs. Note that this includes teh special case
            // of sending to zero outputs
            @SuppressWarnings({ "unchecked", "rawtypes" }) Output<StreamRecord<T>>[] asArray = new Output[allOutputs.size()];
            for (int i = 0; i < allOutputs.size(); i++) {
                asArray[i] = allOutputs.get(i).f0;
            }
            // otherwise multi-chaining would not work correctly.
            if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
                return new CopyingBroadcastingOutputCollector<>(asArray, this);
            } else {
                return new BroadcastingOutputCollector<>(asArray, this);
            }
        }
    } else {
        // otherwise multi-chaining would not work correctly.
        if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
            return new CopyingDirectedOutput<>(selectors, allOutputs);
        } else {
            return new DirectedOutput<>(selectors, allOutputs);
        }
    }
}

Example 5 with RecordWriterOutput

use of org.apache.flink.streaming.runtime.io.RecordWriterOutput in project flink by apache.

the class OperatorChain method createOutputCollector.

private <T> WatermarkGaugeExposingOutput<StreamRecord<T>> createOutputCollector(StreamTask<?, ?> containingTask, StreamConfig operatorConfig, Map<Integer, StreamConfig> chainedConfigs, ClassLoader userCodeClassloader, Map<StreamEdge, RecordWriterOutput<?>> streamOutputs, List<StreamOperatorWrapper<?, ?>> allOperatorWrappers, MailboxExecutorFactory mailboxExecutorFactory) {
    List<Tuple2<WatermarkGaugeExposingOutput<StreamRecord<T>>, StreamEdge>> allOutputs = new ArrayList<>(4);
    // create collectors for the network outputs
    for (StreamEdge outputEdge : operatorConfig.getNonChainedOutputs(userCodeClassloader)) {
        @SuppressWarnings("unchecked") RecordWriterOutput<T> output = (RecordWriterOutput<T>) streamOutputs.get(outputEdge);
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    // Create collectors for the chained outputs
    for (StreamEdge outputEdge : operatorConfig.getChainedOutputs(userCodeClassloader)) {
        int outputId = outputEdge.getTargetId();
        StreamConfig chainedOpConfig = chainedConfigs.get(outputId);
        WatermarkGaugeExposingOutput<StreamRecord<T>> output = createOperatorChain(containingTask, chainedOpConfig, chainedConfigs, userCodeClassloader, streamOutputs, allOperatorWrappers, outputEdge.getOutputTag(), mailboxExecutorFactory);
        allOutputs.add(new Tuple2<>(output, outputEdge));
    }
    if (allOutputs.size() == 1) {
        return allOutputs.get(0).f0;
    } else {
        // send to N outputs. Note that this includes the special case
        // of sending to zero outputs
        @SuppressWarnings({ "unchecked" }) Output<StreamRecord<T>>[] asArray = new Output[allOutputs.size()];
        for (int i = 0; i < allOutputs.size(); i++) {
            asArray[i] = allOutputs.get(i).f0;
        }
        // otherwise multi-chaining would not work correctly.
        if (containingTask.getExecutionConfig().isObjectReuseEnabled()) {
            return closer.register(new CopyingBroadcastingOutputCollector<>(asArray));
        } else {
            return closer.register(new BroadcastingOutputCollector<>(asArray));
        }
    }
}