Search in sources :

Example 1 with SubpartitionIndexRange

use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.

the class SingleInputGateFactory method create.

/**
 * Creates an input gate and all of its input channels.
 */
public SingleInputGate create(@Nonnull ShuffleIOOwnerContext owner, int gateIndex, @Nonnull InputGateDeploymentDescriptor igdd, @Nonnull PartitionProducerStateProvider partitionProducerStateProvider) {
    SupplierWithException<BufferPool, IOException> bufferPoolFactory = createBufferPoolFactory(networkBufferPool, floatingNetworkBuffersPerGate);
    BufferDecompressor bufferDecompressor = null;
    if (igdd.getConsumedPartitionType().isBlocking() && blockingShuffleCompressionEnabled) {
        bufferDecompressor = new BufferDecompressor(networkBufferSize, compressionCodec);
    }
    final String owningTaskName = owner.getOwnerName();
    final MetricGroup networkInputGroup = owner.getInputGroup();
    SubpartitionIndexRange subpartitionIndexRange = igdd.getConsumedSubpartitionIndexRange();
    SingleInputGate inputGate = new SingleInputGate(owningTaskName, gateIndex, igdd.getConsumedResultId(), igdd.getConsumedPartitionType(), subpartitionIndexRange, calculateNumChannels(igdd.getShuffleDescriptors().length, subpartitionIndexRange), partitionProducerStateProvider, bufferPoolFactory, bufferDecompressor, networkBufferPool, networkBufferSize, new ThroughputCalculator(SystemClock.getInstance()), maybeCreateBufferDebloater(gateIndex, networkInputGroup.addGroup(gateIndex)));
    InputChannelMetrics metrics = new InputChannelMetrics(networkInputGroup, owner.getParentGroup());
    createInputChannels(owningTaskName, igdd, inputGate, subpartitionIndexRange, metrics);
    return inputGate;
}
Also used : NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) BufferPool(org.apache.flink.runtime.io.network.buffer.BufferPool) MetricGroup(org.apache.flink.metrics.MetricGroup) ThroughputCalculator(org.apache.flink.runtime.throughput.ThroughputCalculator) IOException(java.io.IOException) InputChannelMetrics(org.apache.flink.runtime.io.network.metrics.InputChannelMetrics) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) BufferDecompressor(org.apache.flink.runtime.io.network.buffer.BufferDecompressor)

Example 2 with SubpartitionIndexRange

use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.

the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.

@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
    IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
    SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
    final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
    ResourceID localLocation = ResourceID.generate();
    SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
    for (InputChannel channel : gate.getInputChannels().values()) {
        if (channel instanceof ChannelStateHolder) {
            ((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
        }
    }
    SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
    SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
    SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
    SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
    SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
    SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
    assertThat(gate.getInputChannels().size(), is(6));
    assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
    assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
    assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
    assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    // test setup
    gate.setup();
    assertNotNull(gate.getBufferPool());
    assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
    gate.finishReadRecoveredState();
    while (!gate.getStateConsumedFuture().isDone()) {
        gate.pollNext();
    }
    // test request partitions
    gate.requestPartitions();
    gate.pollNext();
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
    for (InputChannel inputChannel : gate.getInputChannels().values()) {
        if (inputChannel instanceof RemoteInputChannel) {
            assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
            assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
        } else if (inputChannel instanceof LocalInputChannel) {
            assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
        }
    }
    // test update channels
    gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
    assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
    assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}
Also used : TestingConnectionManager(org.apache.flink.runtime.io.network.TestingConnectionManager) SubpartitionInfo(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) ChannelStateHolder(org.apache.flink.runtime.io.network.partition.ChannelStateHolder) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) NoOpResultSubpartitionView(org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) InputChannelTestUtils.createRemoteInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) InputChannelTestUtils.createLocalInputChannel(org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) IntermediateResultPartitionID(org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID) Test(org.junit.Test)

Example 3 with SubpartitionIndexRange

use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.

the class SsgNetworkMemoryCalculationUtils method getMaxInputChannelNumsForDynamicGraph.

@VisibleForTesting
static Map<IntermediateDataSetID, Integer> getMaxInputChannelNumsForDynamicGraph(ExecutionJobVertex ejv) {
    Map<IntermediateDataSetID, Integer> ret = new HashMap<>();
    for (ExecutionVertex vertex : ejv.getTaskVertices()) {
        for (ConsumedPartitionGroup partitionGroup : vertex.getAllConsumedPartitionGroups()) {
            IntermediateResultPartition resultPartition = ejv.getGraph().getResultPartitionOrThrow((partitionGroup.getFirst()));
            SubpartitionIndexRange subpartitionIndexRange = TaskDeploymentDescriptorFactory.computeConsumedSubpartitionRange(resultPartition, vertex.getParallelSubtaskIndex());
            ret.merge(partitionGroup.getIntermediateDataSetID(), subpartitionIndexRange.size() * partitionGroup.size(), Integer::max);
        }
    }
    return ret;
}
Also used : ConsumedPartitionGroup(org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup) IntermediateResultPartition(org.apache.flink.runtime.executiongraph.IntermediateResultPartition) HashMap(java.util.HashMap) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) SubpartitionIndexRange(org.apache.flink.runtime.deployment.SubpartitionIndexRange) ExecutionVertex(org.apache.flink.runtime.executiongraph.ExecutionVertex) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting)

Aggregations

SubpartitionIndexRange (org.apache.flink.runtime.deployment.SubpartitionIndexRange)3 IOException (java.io.IOException)1 HashMap (java.util.HashMap)1 VisibleForTesting (org.apache.flink.annotation.VisibleForTesting)1 MetricGroup (org.apache.flink.metrics.MetricGroup)1 ResourceID (org.apache.flink.runtime.clusterframework.types.ResourceID)1 ExecutionVertex (org.apache.flink.runtime.executiongraph.ExecutionVertex)1 IntermediateResultPartition (org.apache.flink.runtime.executiongraph.IntermediateResultPartition)1 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)1 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)1 TestingConnectionManager (org.apache.flink.runtime.io.network.TestingConnectionManager)1 BufferDecompressor (org.apache.flink.runtime.io.network.buffer.BufferDecompressor)1 BufferPool (org.apache.flink.runtime.io.network.buffer.BufferPool)1 NetworkBufferPool (org.apache.flink.runtime.io.network.buffer.NetworkBufferPool)1 InputChannelMetrics (org.apache.flink.runtime.io.network.metrics.InputChannelMetrics)1 ChannelStateHolder (org.apache.flink.runtime.io.network.partition.ChannelStateHolder)1 InputChannelTestUtils.createLocalInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel)1 InputChannelTestUtils.createRemoteInputChannel (org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createRemoteInputChannel)1 NoOpResultSubpartitionView (org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView)1 SubpartitionInfo (org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate.SubpartitionInfo)1