Examples with IntermediateDataSetID org.apache.flink.runtime.jobgraph.IntermediateDataSetID used on opensource projects

Example 31 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionLocalChannels.

@Test
public void testFairConsumptionLocalChannels() throws Exception {
    final int numChannels = 37;
    final int buffersPerChannel = 27;
    final ResultPartition resultPartition = mock(ResultPartition.class);
    final Buffer mockBuffer = createMockBuffer(42);
    // ----- create some source channels and fill them with one buffer each -----
    final PipelinedSubpartition[] sources = new PipelinedSubpartition[numChannels];
    for (int i = 0; i < numChannels; i++) {
        sources[i] = new PipelinedSubpartition(0, resultPartition);
    }
    // ----- create reading side -----
    ResultPartitionManager resultPartitionManager = createResultPartitionManager(sources);
    SingleInputGate gate = new FairnessVerifyingInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), 0, numChannels, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    for (int i = 0; i < numChannels; i++) {
        LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(), resultPartitionManager, mock(TaskEventDispatcher.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
        gate.setInputChannel(new IntermediateResultPartitionID(), channel);
    }
    // seed one initial buffer
    sources[12].add(mockBuffer);
    // read all the buffers and the EOF event
    for (int i = 0; i < numChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNextBufferOrEvent());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getCurrentNumberOfBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == min + 1);
        if (i % (2 * numChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(sources, 3, mockBuffer);
        }
    }
// there is still more in the queues
}

Example 32 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class SingleInputGateTest method testBasicGetNextLogic.

/**
	 * Tests basic correctness of buffer-or-event interleaving and correct <code>null</code> return
	 * value after receiving all end-of-partition events.
	 */
@Test(timeout = 120 * 1000)
public void testBasicGetNextLogic() throws Exception {
    // Setup
    final SingleInputGate inputGate = new SingleInputGate("Test Task Name", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 2, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    assertEquals(ResultPartitionType.PIPELINED, inputGate.getConsumedPartitionType());
    final TestInputChannel[] inputChannels = new TestInputChannel[] { new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1) };
    inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannels[0].getInputChannel());
    inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannels[1].getInputChannel());
    // Test
    inputChannels[0].readBuffer();
    inputChannels[0].readBuffer();
    inputChannels[1].readBuffer();
    inputChannels[1].readEndOfPartitionEvent();
    inputChannels[0].readEndOfPartitionEvent();
    inputGate.notifyChannelNonEmpty(inputChannels[0].getInputChannel());
    inputGate.notifyChannelNonEmpty(inputChannels[1].getInputChannel());
    verifyBufferOrEvent(inputGate, true, 0);
    verifyBufferOrEvent(inputGate, true, 1);
    verifyBufferOrEvent(inputGate, true, 0);
    verifyBufferOrEvent(inputGate, false, 1);
    verifyBufferOrEvent(inputGate, false, 0);
    // Return null when the input gate has received all end-of-partition events
    assertTrue(inputGate.isFinished());
    assertNull(inputGate.getNextBufferOrEvent());
}

Example 33 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class SingleInputGateTest method testReleaseWhilePollingChannel.

/**
	 * Tests that the release of the input gate is noticed while polling the
	 * channels for available data.
	 */
@Test
public void testReleaseWhilePollingChannel() throws Exception {
    final AtomicReference<Exception> asyncException = new AtomicReference<>();
    // Setup the input gate with a single channel that does nothing
    final SingleInputGate inputGate = new SingleInputGate("InputGate", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 1, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    InputChannel unknown = new UnknownInputChannel(inputGate, 0, new ResultPartitionID(), new ResultPartitionManager(), new TaskEventDispatcher(), new LocalConnectionManager(), 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    inputGate.setInputChannel(unknown.partitionId.getPartitionId(), unknown);
    // Start the consumer in a separate Thread
    Thread asyncConsumer = new Thread() {

        @Override
        public void run() {
            try {
                inputGate.getNextBufferOrEvent();
            } catch (Exception e) {
                asyncException.set(e);
            }
        }
    };
    asyncConsumer.start();
    // Wait for blocking queue poll call and release input gate
    boolean success = false;
    for (int i = 0; i < 50; i++) {
        if (asyncConsumer.isAlive()) {
            success = asyncConsumer.getState() == Thread.State.WAITING;
        }
        if (success) {
            break;
        } else {
            // Retry
            Thread.sleep(100);
        }
    }
    // Verify that async consumer is in blocking request
    assertTrue("Did not trigger blocking buffer request.", success);
    // Release the input gate
    inputGate.releaseAllResources();
    // Wait for Thread to finish and verify expected Exceptions. If the
    // input gate status is not properly checked during requests, this
    // call will never return.
    asyncConsumer.join();
    assertNotNull(asyncException.get());
    assertEquals(IllegalStateException.class, asyncException.get().getClass());
}

Example 34 with IntermediateDataSetID

use of org.apache.flink.runtime.jobgraph.IntermediateDataSetID in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
	 * Tests request back off configuration is correctly forwarded to the channels.
	 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    ResultPartitionID[] partitionIds = new ResultPartitionID[] { new ResultPartitionID(), new ResultPartitionID(), new ResultPartitionID() };
    InputChannelDeploymentDescriptor[] channelDescs = new InputChannelDeploymentDescriptor[] { // Local
    new InputChannelDeploymentDescriptor(partitionIds[0], ResultPartitionLocation.createLocal()), // Remote
    new InputChannelDeploymentDescriptor(partitionIds[1], ResultPartitionLocation.createRemote(new ConnectionID(new InetSocketAddress("localhost", 5000), 0))), // Unknown
    new InputChannelDeploymentDescriptor(partitionIds[2], ResultPartitionLocation.createUnknown()) };
    InputGateDeploymentDescriptor gateDesc = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, channelDescs);
    int initialBackoff = 137;
    int maxBackoff = 1001;
    NetworkEnvironment netEnv = mock(NetworkEnvironment.class);
    when(netEnv.getResultPartitionManager()).thenReturn(new ResultPartitionManager());
    when(netEnv.getTaskEventDispatcher()).thenReturn(new TaskEventDispatcher());
    when(netEnv.getPartitionRequestInitialBackoff()).thenReturn(initialBackoff);
    when(netEnv.getPartitionRequestMaxBackoff()).thenReturn(maxBackoff);
    when(netEnv.getConnectionManager()).thenReturn(new LocalConnectionManager());
    SingleInputGate gate = SingleInputGate.create("TestTask", new JobID(), new ExecutionAttemptID(), gateDesc, netEnv, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    assertEquals(gateDesc.getConsumedPartitionType(), gate.getConsumedPartitionType());
    Map<IntermediateResultPartitionID, InputChannel> channelMap = gate.getInputChannels();
    assertEquals(3, channelMap.size());
    InputChannel localChannel = channelMap.get(partitionIds[0].getPartitionId());
    assertEquals(LocalInputChannel.class, localChannel.getClass());
    InputChannel remoteChannel = channelMap.get(partitionIds[1].getPartitionId());
    assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
    InputChannel unknownChannel = channelMap.get(partitionIds[2].getPartitionId());
    assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
    InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
    for (InputChannel ch : channels) {
        assertEquals(0, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(maxBackoff, ch.getCurrentBackoff());
        assertFalse(ch.increaseBackoff());
    }
}