use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.
the class SingleInputGateFactory method create.
/**
* Creates an input gate and all of its input channels.
*/
public SingleInputGate create(@Nonnull ShuffleIOOwnerContext owner, int gateIndex, @Nonnull InputGateDeploymentDescriptor igdd, @Nonnull PartitionProducerStateProvider partitionProducerStateProvider) {
SupplierWithException<BufferPool, IOException> bufferPoolFactory = createBufferPoolFactory(networkBufferPool, floatingNetworkBuffersPerGate);
BufferDecompressor bufferDecompressor = null;
if (igdd.getConsumedPartitionType().isBlocking() && blockingShuffleCompressionEnabled) {
bufferDecompressor = new BufferDecompressor(networkBufferSize, compressionCodec);
}
final String owningTaskName = owner.getOwnerName();
final MetricGroup networkInputGroup = owner.getInputGroup();
SubpartitionIndexRange subpartitionIndexRange = igdd.getConsumedSubpartitionIndexRange();
SingleInputGate inputGate = new SingleInputGate(owningTaskName, gateIndex, igdd.getConsumedResultId(), igdd.getConsumedPartitionType(), subpartitionIndexRange, calculateNumChannels(igdd.getShuffleDescriptors().length, subpartitionIndexRange), partitionProducerStateProvider, bufferPoolFactory, bufferDecompressor, networkBufferPool, networkBufferSize, new ThroughputCalculator(SystemClock.getInstance()), maybeCreateBufferDebloater(gateIndex, networkInputGroup.addGroup(gateIndex)));
InputChannelMetrics metrics = new InputChannelMetrics(networkInputGroup, owner.getParentGroup());
createInputChannels(owningTaskName, igdd, inputGate, subpartitionIndexRange, metrics);
return inputGate;
}
use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.
the class SingleInputGateTest method testSingleInputGateWithSubpartitionIndexRange.
@Test
public void testSingleInputGateWithSubpartitionIndexRange() throws IOException, InterruptedException {
IntermediateResultPartitionID[] partitionIds = new IntermediateResultPartitionID[] { new IntermediateResultPartitionID(), new IntermediateResultPartitionID(), new IntermediateResultPartitionID() };
SubpartitionIndexRange subpartitionIndexRange = new SubpartitionIndexRange(0, 1);
final NettyShuffleEnvironment netEnv = new NettyShuffleEnvironmentBuilder().build();
ResourceID localLocation = ResourceID.generate();
SingleInputGate gate = createSingleInputGate(partitionIds, ResultPartitionType.BLOCKING, subpartitionIndexRange, netEnv, localLocation, new TestingConnectionManager(), new TestingResultPartitionManager(new NoOpResultSubpartitionView()));
for (InputChannel channel : gate.getInputChannels().values()) {
if (channel instanceof ChannelStateHolder) {
((ChannelStateHolder) channel).setChannelStateWriter(ChannelStateWriter.NO_OP);
}
}
SubpartitionInfo info1 = createSubpartitionInfo(partitionIds[0], 0);
SubpartitionInfo info2 = createSubpartitionInfo(partitionIds[0], 1);
SubpartitionInfo info3 = createSubpartitionInfo(partitionIds[1], 0);
SubpartitionInfo info4 = createSubpartitionInfo(partitionIds[1], 1);
SubpartitionInfo info5 = createSubpartitionInfo(partitionIds[2], 0);
SubpartitionInfo info6 = createSubpartitionInfo(partitionIds[2], 1);
assertThat(gate.getInputChannels().size(), is(6));
assertThat(gate.getInputChannels().get(info1).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info2).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info3).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info4).getConsumedSubpartitionIndex(), is(1));
assertThat(gate.getInputChannels().get(info5).getConsumedSubpartitionIndex(), is(0));
assertThat(gate.getInputChannels().get(info6).getConsumedSubpartitionIndex(), is(1));
assertChannelsType(gate, LocalRecoveredInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteRecoveredInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
// test setup
gate.setup();
assertNotNull(gate.getBufferPool());
assertEquals(1, gate.getBufferPool().getNumberOfRequiredMemorySegments());
gate.finishReadRecoveredState();
while (!gate.getStateConsumedFuture().isDone()) {
gate.pollNext();
}
// test request partitions
gate.requestPartitions();
gate.pollNext();
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, UnknownInputChannel.class, Arrays.asList(info5, info6));
for (InputChannel inputChannel : gate.getInputChannels().values()) {
if (inputChannel instanceof RemoteInputChannel) {
assertNotNull(((RemoteInputChannel) inputChannel).getPartitionRequestClient());
assertEquals(2, ((RemoteInputChannel) inputChannel).getInitialCredit());
} else if (inputChannel instanceof LocalInputChannel) {
assertNotNull(((LocalInputChannel) inputChannel).getSubpartitionView());
}
}
// test update channels
gate.updateInputChannel(localLocation, createRemoteWithIdAndLocation(partitionIds[2], localLocation));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info1, info2));
assertChannelsType(gate, RemoteInputChannel.class, Arrays.asList(info3, info4));
assertChannelsType(gate, LocalInputChannel.class, Arrays.asList(info5, info6));
}
use of org.apache.flink.runtime.deployment.SubpartitionIndexRange in project flink by apache.
the class SsgNetworkMemoryCalculationUtils method getMaxInputChannelNumsForDynamicGraph.
@VisibleForTesting
static Map<IntermediateDataSetID, Integer> getMaxInputChannelNumsForDynamicGraph(ExecutionJobVertex ejv) {
Map<IntermediateDataSetID, Integer> ret = new HashMap<>();
for (ExecutionVertex vertex : ejv.getTaskVertices()) {
for (ConsumedPartitionGroup partitionGroup : vertex.getAllConsumedPartitionGroups()) {
IntermediateResultPartition resultPartition = ejv.getGraph().getResultPartitionOrThrow((partitionGroup.getFirst()));
SubpartitionIndexRange subpartitionIndexRange = TaskDeploymentDescriptorFactory.computeConsumedSubpartitionRange(resultPartition, vertex.getParallelSubtaskIndex());
ret.merge(partitionGroup.getIntermediateDataSetID(), subpartitionIndexRange.size() * partitionGroup.size(), Integer::max);
}
}
return ret;
}
Aggregations