Examples with SubtaskConnectionDescriptor org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor used on opensource projects

Example 1 with SubtaskConnectionDescriptor

use of org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor in project flink by apache.

the class ResultSubpartitionRecoveredStateHandler method recover.

@Override
public void recover(ResultSubpartitionInfo subpartitionInfo, int oldSubtaskIndex, BufferWithContext<BufferBuilder> bufferWithContext) throws IOException {
    try (BufferBuilder bufferBuilder = bufferWithContext.context) {
        try (BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumerFromBeginning()) {
            bufferBuilder.finish();
            if (bufferConsumer.isDataAvailable()) {
                final List<CheckpointedResultSubpartition> channels = getMappedChannels(subpartitionInfo);
                for (final CheckpointedResultSubpartition channel : channels) {
                    // channel selector is created from the downstream's point of view: the
                    // subtask of downstream = subpartition index of recovered buffer
                    final SubtaskConnectionDescriptor channelSelector = new SubtaskConnectionDescriptor(subpartitionInfo.getSubPartitionIdx(), oldSubtaskIndex);
                    channel.addRecovered(EventSerializer.toBufferConsumer(channelSelector, false));
                    channel.addRecovered(bufferConsumer.copy());
                }
            }
        }
    }
}

Example 2 with SubtaskConnectionDescriptor

use of org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor in project flink by apache.

the class DemultiplexingRecordDeserializer method create.

static <T> DemultiplexingRecordDeserializer<T> create(InputChannelInfo channelInfo, InflightDataRescalingDescriptor rescalingDescriptor, Function<Integer, RecordDeserializer<DeserializationDelegate<StreamElement>>> deserializerFactory, Function<InputChannelInfo, Predicate<StreamRecord<T>>> recordFilterFactory) {
    int[] oldSubtaskIndexes = rescalingDescriptor.getOldSubtaskIndexes(channelInfo.getGateIdx());
    if (oldSubtaskIndexes.length == 0) {
        return UNMAPPED;
    }
    final int[] oldChannelIndexes = rescalingDescriptor.getChannelMapping(channelInfo.getGateIdx()).getMappedIndexes(channelInfo.getInputChannelIdx());
    if (oldChannelIndexes.length == 0) {
        return UNMAPPED;
    }
    int totalChannels = oldSubtaskIndexes.length * oldChannelIndexes.length;
    Map<SubtaskConnectionDescriptor, VirtualChannel<T>> virtualChannels = Maps.newHashMapWithExpectedSize(totalChannels);
    for (int subtask : oldSubtaskIndexes) {
        for (int channel : oldChannelIndexes) {
            SubtaskConnectionDescriptor descriptor = new SubtaskConnectionDescriptor(subtask, channel);
            virtualChannels.put(descriptor, new VirtualChannel<>(deserializerFactory.apply(totalChannels), rescalingDescriptor.isAmbiguous(channelInfo.getGateIdx(), subtask) ? recordFilterFactory.apply(channelInfo) : RecordFilter.all()));
        }
    }
    return new DemultiplexingRecordDeserializer(virtualChannels);
}

Example 3 with SubtaskConnectionDescriptor

use of org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor in project flink by apache.

the class DemultiplexingRecordDeserializerTest method testWatermarks.

/**
 * Tests that Watermarks are only forwarded when all watermarks are received.
 */
@Test
public void testWatermarks() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(0, 0), rescalingDescriptor(to(0, 1), array(mappings(to(0, 1), to(4, 5))), emptySet()), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> RecordFilter.all());
    assertEquals(4, deserializer.getVirtualChannelSelectors().size());
    for (Iterator<SubtaskConnectionDescriptor> iterator = deserializer.getVirtualChannelSelectors().iterator(); iterator.hasNext(); ) {
        SubtaskConnectionDescriptor selector = iterator.next();
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            final long ts = 42L + selector.getInputSubtaskIndex() + selector.getOutputSubtaskIndex();
            Buffer buffer = write(bufferBuilder, new Watermark(ts));
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
        }
        if (iterator.hasNext()) {
            assertEquals(Collections.emptyList(), read(deserializer));
        } else {
            // last channel, min should be 42 + 0 + 0
            assertEquals(Arrays.asList(new Watermark(42)), read(deserializer));
        }
        assertTrue(memorySegment.isFreed());
    }
}

Example 4 with SubtaskConnectionDescriptor

use of org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor in project flink by apache.

the class DemultiplexingRecordDeserializerTest method testAmbiguousChannels.

/**
 * Tests that {@link RecordFilter} are used correctly.
 */
@Test
public void testAmbiguousChannels() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(1, 0), rescalingDescriptor(to(41, 42), array(mappings(), mappings(to(2, 3), to(4, 5))), set(42)), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> new RecordFilter(new ModSelector(2), LongSerializer.INSTANCE, 1));
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(41, 2), new SubtaskConnectionDescriptor(41, 3), new SubtaskConnectionDescriptor(42, 2), new SubtaskConnectionDescriptor(42, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            // add one even and one odd number
            Buffer buffer = writeLongs(bufferBuilder, i, i + 1L);
            SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), i / 10 % 2);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
            if (selector.getInputSubtaskIndex() == 41) {
                assertEquals(Arrays.asList((long) i, i + 1L), readLongs(deserializer));
            } else {
                // only odd should occur in output
                assertEquals(Arrays.asList(i / 2 * 2 + 1L), readLongs(deserializer));
            }
        }
        assertTrue(memorySegment.isFreed());
    }
}

Example 5 with SubtaskConnectionDescriptor

use of org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor in project flink by apache.

the class DemultiplexingRecordDeserializerTest method testUpscale.

/**
 * Tests {@link SubtaskConnectionDescriptor} by mixing buffers from 4 different virtual
 * channels.
 */
@Test
public void testUpscale() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(2, 0), rescalingDescriptor(to(0, 1), array(mappings(), mappings(), mappings(to(2, 3), to(4, 5))), emptySet()), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> RecordFilter.all());
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(0, 2), new SubtaskConnectionDescriptor(0, 3), new SubtaskConnectionDescriptor(1, 2), new SubtaskConnectionDescriptor(1, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), random.nextInt(4));
        long start = selector.getInputSubtaskIndex() << 4 | selector.getOutputSubtaskIndex();
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            Buffer buffer = writeLongs(bufferBuilder, start + 1L, start + 2L, start + 3L);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
        }
        assertEquals(Arrays.asList(start + 1L, start + 2L, start + 3L), readLongs(deserializer));
        assertTrue(memorySegment.isFreed());
    }
}