use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.
the class ResultSubpartitionRecoveredStateHandler method recover.
@Override
public void recover(ResultSubpartitionInfo subpartitionInfo, int oldSubtaskIndex, BufferWithContext<BufferBuilder> bufferWithContext) throws IOException {
try (BufferBuilder bufferBuilder = bufferWithContext.context) {
try (BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumerFromBeginning()) {
bufferBuilder.finish();
if (bufferConsumer.isDataAvailable()) {
final List<CheckpointedResultSubpartition> channels = getMappedChannels(subpartitionInfo);
for (final CheckpointedResultSubpartition channel : channels) {
// channel selector is created from the downstream's point of view: the
// subtask of downstream = subpartition index of recovered buffer
final SubtaskConnectionDescriptor channelSelector = new SubtaskConnectionDescriptor(subpartitionInfo.getSubPartitionIdx(), oldSubtaskIndex);
channel.addRecovered(EventSerializer.toBufferConsumer(channelSelector, false));
channel.addRecovered(bufferConsumer.copy());
}
}
}
}
}
use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.
the class StreamTaskNetworkInputTest method createDataBuffer.
private BufferOrEvent createDataBuffer() throws IOException {
try (BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE)) {
BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
serializeRecord(42L, bufferBuilder);
serializeRecord(44L, bufferBuilder);
return new BufferOrEvent(bufferConsumer.build(), new InputChannelInfo(0, 0));
}
}
use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.
the class DemultiplexingRecordDeserializerTest method testAmbiguousChannels.
/**
* Tests that {@link RecordFilter} are used correctly.
*/
@Test
public void testAmbiguousChannels() throws IOException {
DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(1, 0), rescalingDescriptor(to(41, 42), array(mappings(), mappings(to(2, 3), to(4, 5))), set(42)), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> new RecordFilter(new ModSelector(2), LongSerializer.INSTANCE, 1));
assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(41, 2), new SubtaskConnectionDescriptor(41, 3), new SubtaskConnectionDescriptor(42, 2), new SubtaskConnectionDescriptor(42, 3)), deserializer.getVirtualChannelSelectors());
for (int i = 0; i < 100; i++) {
MemorySegment memorySegment = allocateUnpooledSegment(128);
try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
// add one even and one odd number
Buffer buffer = writeLongs(bufferBuilder, i, i + 1L);
SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), i / 10 % 2);
deserializer.select(selector);
deserializer.setNextBuffer(buffer);
if (selector.getInputSubtaskIndex() == 41) {
assertEquals(Arrays.asList((long) i, i + 1L), readLongs(deserializer));
} else {
// only odd should occur in output
assertEquals(Arrays.asList(i / 2 * 2 + 1L), readLongs(deserializer));
}
}
assertTrue(memorySegment.isFreed());
}
}
use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.
the class IteratorWrappingTestSingleInputGate method wrapIterator.
private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException {
inputIterator = iterator;
serializer = new DataOutputSerializer(128);
// The input iterator can produce an infinite stream. That's why we have to serialize each
// record on demand and cannot do it upfront.
final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() {
private boolean hasData = inputIterator.next(reuse) != null;
@Override
public Optional<BufferAndAvailability> getBufferAvailability() throws IOException {
if (hasData) {
ByteBuffer serializedRecord = RecordWriter.serializeRecord(serializer, reuse);
BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
bufferBuilder.appendAndCommit(serializedRecord);
hasData = inputIterator.next(reuse) != null;
// Call getCurrentBuffer to ensure size is set
final Buffer.DataType nextDataType = hasData ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.EVENT_BUFFER;
return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextDataType, 0, 0));
} else {
inputChannel.setReleased();
return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), Buffer.DataType.NONE, 0, 0));
}
}
};
inputChannel.addBufferAndAvailability(answer);
inputGate.setInputChannels(inputChannel);
return this;
}
use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.
the class StreamTestSingleInputGate method setupInputChannels.
private TestInputChannel[] setupInputChannels() {
TestInputChannel[] inputChannels = new TestInputChannel[numInputChannels];
for (int i = 0; i < numInputChannels; i++) {
final int channelIndex = i;
final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<T>(serializer));
inputQueues[channelIndex] = new ConcurrentLinkedQueue<>();
inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
final BufferAndAvailabilityProvider answer = () -> {
ConcurrentLinkedQueue<InputValue<Object>> inputQueue = inputQueues[channelIndex];
InputValue<Object> input;
Buffer.DataType nextType;
synchronized (inputQueue) {
input = inputQueue.poll();
nextType = !inputQueue.isEmpty() ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.NONE;
}
if (input != null && input.isStreamEnd()) {
inputChannels[channelIndex].setReleased();
return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), nextType, 0, 0));
} else if (input != null && input.isDataEnd()) {
return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(new EndOfData(StopMode.DRAIN), false), nextType, 0, 0));
} else if (input != null && input.isStreamRecord()) {
StreamElement inputElement = input.getStreamRecord();
delegate.setInstance(inputElement);
ByteBuffer serializedRecord = RecordWriter.serializeRecord(dataOutputSerializer, delegate);
BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
bufferBuilder.appendAndCommit(serializedRecord);
bufferBuilder.finish();
bufferBuilder.close();
// Call getCurrentBuffer to ensure size is set
return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextType, 0, 0));
} else if (input != null && input.isEvent()) {
AbstractEvent event = input.getEvent();
if (event instanceof EndOfPartitionEvent) {
inputChannels[channelIndex].setReleased();
}
return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event, false), nextType, 0, 0));
} else {
return Optional.empty();
}
};
inputChannels[channelIndex].addBufferAndAvailability(answer);
}
return inputChannels;
}
Aggregations