Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 36 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class TaskExecutor method updatePartitions.

// ----------------------------------------------------------------------
// Partition lifecycle RPCs
// ----------------------------------------------------------------------
@RpcMethod
public Acknowledge updatePartitions(final ExecutionAttemptID executionAttemptID, Iterable<PartitionInfo> partitionInfos) throws PartitionException {
    final Task task = taskSlotTable.getTask(executionAttemptID);
    if (task != null) {
        for (final PartitionInfo partitionInfo : partitionInfos) {
            IntermediateDataSetID intermediateResultPartitionID = partitionInfo.getIntermediateDataSetID();
            final SingleInputGate singleInputGate = task.getInputGateById(intermediateResultPartitionID);
            if (singleInputGate != null) {
                // Run asynchronously because it might be blocking
                getRpcService().execute(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            singleInputGate.updateInputChannel(partitionInfo.getInputChannelDeploymentDescriptor());
                        } catch (IOException | InterruptedException e) {
                            log.error("Could not update input data location for task {}. Trying to fail task.", task.getTaskInfo().getTaskName(), e);
                            try {
                                task.failExternally(e);
                            } catch (RuntimeException re) {
                                // TODO: Check whether we need this or make exception in failExtenally checked
                                log.error("Failed canceling task with execution ID {} after task update failure.", executionAttemptID, re);
                            }
                        }
                    }
                });
            } else {
                throw new PartitionException("No reader with ID " + intermediateResultPartitionID + " for task " + executionAttemptID + " was found.");
            }
        }
        return Acknowledge.get();
    } else {
        log.debug("Discard update for input partitions of task {}. Task is no longer running.", executionAttemptID);
        return Acknowledge.get();
    }
}

Example 37 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateConcurrentTest method testConsumptionWithMixedChannels.

@Test
public void testConsumptionWithMixedChannels() throws Exception {
    final int numChannels = 61;
    final int numLocalChannels = 20;
    final int buffersPerChannel = 1000;
    // fill the local/remote decision
    List<Boolean> localOrRemote = new ArrayList<>(numChannels);
    for (int i = 0; i < numChannels; i++) {
        localOrRemote.add(i < numLocalChannels);
    }
    Collections.shuffle(localOrRemote);
    final ConnectionManager connManager = createDummyConnectionManager();
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final PipelinedSubpartition[] localPartitions = new PipelinedSubpartition[numLocalChannels];
    final ResultPartitionManager resultPartitionManager = createResultPartitionManager(localPartitions);
    final Source[] sources = new Source[numChannels];
    final SingleInputGate gate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0, local = 0; i < numChannels; i++) {
        if (localOrRemote.get(i)) {
            // local channel
            PipelinedSubpartition psp = new PipelinedSubpartition(0, resultPartition);
            localPartitions[local++] = psp;
            sources[i] = new PipelinedSubpartitionSource(psp);
            LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
            gate.setInputChannel(new IntermediateResultPartitionID(), channel);
        } else {
            //remote channel
            RemoteInputChannel channel = new RemoteInputChannel(gate, i, new ResultPartitionID(), mock(ConnectionID.class), connManager, 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
            gate.setInputChannel(new IntermediateResultPartitionID(), channel);
            sources[i] = new RemoteChannelSource(channel);
        }
    }
    ProducerThread producer = new ProducerThread(sources, numChannels * buffersPerChannel, 4, 10);
    ConsumerThread consumer = new ConsumerThread(gate, numChannels * buffersPerChannel);
    producer.start();
    consumer.start();
    // the 'sync()' call checks for exceptions and failed assertions
    producer.sync();
    consumer.sync();
}

Example 38 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyShuffleUtilsTest method testComputeRequiredNetworkBuffers.

/**
 * This test verifies that the {@link NettyShuffleEnvironment} requires buffers as expected, so
 * that the required shuffle memory size returned by {@link
 * ShuffleMaster#computeShuffleMemorySizeForTask(TaskInputsOutputsDescriptor)} is correct.
 */
@Test
public void testComputeRequiredNetworkBuffers() throws Exception {
    int numBuffersPerChannel = 5;
    int numBuffersPerGate = 8;
    int sortShuffleMinParallelism = 8;
    int numSortShuffleMinBuffers = 12;
    int numChannels1 = 3;
    int numChannels2 = 4;
    IntermediateDataSetID ds1 = new IntermediateDataSetID();
    IntermediateDataSetID ds2 = new IntermediateDataSetID();
    IntermediateDataSetID ds3 = new IntermediateDataSetID();
    // pipelined shuffle
    int numSubs1 = 5;
    // hash blocking shuffle
    int numSubs2 = 6;
    // sort blocking shuffle
    int numSubs3 = 10;
    Map<IntermediateDataSetID, Integer> subpartitionNums = ImmutableMap.of(ds1, numSubs1, ds2, numSubs2, ds3, numSubs3);
    Map<IntermediateDataSetID, ResultPartitionType> partitionTypes = ImmutableMap.of(ds1, PIPELINED_BOUNDED, ds2, BLOCKING, ds3, BLOCKING);
    int numTotalBuffers = NettyShuffleUtils.computeNetworkBuffersForAnnouncing(numBuffersPerChannel, numBuffersPerGate, sortShuffleMinParallelism, numSortShuffleMinBuffers, numChannels1 + numChannels2, 2, subpartitionNums, partitionTypes);
    NettyShuffleEnvironment sEnv = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(numTotalBuffers).setNetworkBuffersPerChannel(numBuffersPerChannel).setSortShuffleMinBuffers(numSortShuffleMinBuffers).setSortShuffleMinParallelism(sortShuffleMinParallelism).build();
    SingleInputGate inputGate1 = createInputGate(sEnv, PIPELINED_BOUNDED, numChannels1);
    inputGate1.setup();
    SingleInputGate inputGate2 = createInputGate(sEnv, BLOCKING, numChannels2);
    inputGate2.setup();
    ResultPartition resultPartition1 = createResultPartition(sEnv, PIPELINED_BOUNDED, numSubs1);
    resultPartition1.setup();
    ResultPartition resultPartition2 = createResultPartition(sEnv, BLOCKING, numSubs2);
    resultPartition2.setup();
    ResultPartition resultPartition3 = createResultPartition(sEnv, BLOCKING, numSubs3);
    resultPartition3.setup();
    int expected = calculateBuffersConsumption(inputGate1) + calculateBuffersConsumption(inputGate2) + calculateBuffersConsumption(resultPartition1) + calculateBuffersConsumption(resultPartition2) + calculateBuffersConsumption(resultPartition3);
    assertEquals(expected, numTotalBuffers);
    inputGate1.close();
    inputGate2.close();
    resultPartition1.close();
    resultPartition2.close();
    resultPartition3.close();
}

Example 39 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class ChannelPersistenceITCase method buildGate.

private SingleInputGate buildGate(NetworkBufferPool networkBufferPool, int numberOfChannels) throws IOException {
    SingleInputGate gate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildRemoteRecoveredChannel).setBufferPoolFactory(networkBufferPool.createBufferPool(numberOfChannels, Integer.MAX_VALUE)).setSegmentProvider(networkBufferPool).setNumberOfChannels(numberOfChannels).build();
    gate.setup();
    return gate;
}

Example 40 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class AlternatingCheckpointsTest method testBarrierHandling.

private void testBarrierHandling(SnapshotType checkpointType) throws Exception {
    final long barrierId = 123L;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    SingleInputGate gate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
    TestInputChannel fast = new TestInputChannel(gate, 0, false, true);
    TestInputChannel slow = new TestInputChannel(gate, 1, false, true);
    gate.setInputChannels(fast, slow);
    SingleCheckpointBarrierHandler barrierHandler = getTestBarrierHandlerFactory(target).create(gate);
    CheckpointedInputGate checkpointedGate = new CheckpointedInputGate(gate, barrierHandler, new SyncMailboxExecutor());
    if (checkpointType.isSavepoint()) {
        fast.setBlocked(true);
        slow.setBlocked(true);
    }
    CheckpointOptions options = checkpointType.isSavepoint() ? alignedNoTimeout(checkpointType, getDefault()) : unaligned(CheckpointType.CHECKPOINT, getDefault());
    Buffer barrier = barrier(barrierId, 1, options);
    send(barrier.retainBuffer(), fast, checkpointedGate);
    assertEquals(checkpointType.isSavepoint(), target.triggeredCheckpoints.isEmpty());
    send(barrier.retainBuffer(), slow, checkpointedGate);
    assertEquals(singletonList(barrierId), target.triggeredCheckpoints);
    if (checkpointType.isSavepoint()) {
        for (InputChannel channel : gate.getInputChannels().values()) {
            assertFalse(String.format("channel %d should be resumed", channel.getChannelIndex()), ((TestInputChannel) channel).isBlocked());
        }
    }
}